Merge branch 'claude/rate-limit-behavior-474ecf'

Strip only a trailing numeric suffix from blog folder names, fixing
zomb-eh_10 importing as blog 'zomb-eh0'.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

AGENTS.md

@@ -11,6 +11,7 @@
 - `ResponseNotes.cs`: Tumblr API response models
 - Round-robin API key rotation with rate-limit tracking
 - Automatic console color assignment per API key for output differentiation
+- No-argument mode (`Program.TraverseDirectory`) ingests `.txt` blog export files into `Posts` via `DataAccess.AddPost`. Recognized field prefixes live in `TraverseDirectoryFieldPrefixes`; `Body:` and `Downloaded files:` collect every following line up to the next recognized prefix (multi-line values). `RootURL` is populated from a `Reblog root url:` line the same way it's populated from the API-based `--likes` flow — both paths converge on `DataAccess.AddPost`'s `rootURL` parameter, which `UpdatePost` only overwrites when the incoming value is non-empty (existing `RootURL` is preserved otherwise)
 
 ## Developer Guidelines
 
@@ -27,6 +28,25 @@
 - Preserve console color state: use save/restore pattern for temporary color changes
 - API rate limits must use `ApiKeyPool.MarkRateLimited()`/`MarkAvailable()`
 
+### API Failure Classification
+Tumblr sits behind a CDN that returns HTML error pages (403, 5xx) which never reach the API. These
+say nothing about the item being fetched, so they must not be recorded as per-item failures.
+
+- A response body that will not parse as JSON did not come from the API. Flag it with
+  `Root.transientFailure`, never as `FAILURE`
+- Transient failures retry in place (`TransientBackoffSeconds`) before the item is skipped; a skipped
+  item stays unmarked in the DB so a later launch retries it
+- `MaxConsecutiveTransient` consecutive transient failures aborts the pass rather than skipping
+  item-by-item against an edge that is refusing all traffic
+- Only call `ApiKeyPool.MarkAvailable()` on a response that actually reached the API. A transport or
+  CDN failure says nothing about the key's standing and must not clear its flag
+- Only a real HTTP 429 (or `meta.status == 429`) counts as a rate limit. Do not infer one from the
+  presence of `X-RateLimit-*` headers, which Tumblr sends on every response
+- Rate limiters must pace with `await AcquireAsync()`. `AttemptAcquire()` does not wait, so a
+  saturated window aborts the run instead of throttling it
+- Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or
+  skipped items), so a caller can distinguish that from a clean run
+
 ### Testing
 - No existing test suite; use xUnit if adding tests
 - Test critical logic: `ApiKeyPool` init, color parsing, config persistence

URLNotesGrabberCORE/DataAccess.cs

@@ -37,6 +37,7 @@ namespace URLNotesGrabberCORE
         public string reblogKey;
         public string reblogName;
         public string reblogURL;
+        public string rootURL;
         public string slug;
         public string summary;
         public string tags;
@@ -60,6 +61,7 @@ namespace URLNotesGrabberCORE
             reblogKey = ".";
             reblogName = ".";
             reblogURL = ".";
+            rootURL = ".";
             slug = ".";
             summary = ".";
             tags = ".";
@@ -2609,6 +2611,10 @@ namespace URLNotesGrabberCORE
 
         public static void MarkAvailable(ApiKeyConfig key)
         {
+            // Called after every successful call; skip the write and the log line when nothing was flagged.
+            if (GetRetryUntil(key) == 0)
+                return;
+
             using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
             conn.Open();
             using var cmd = new System.Data.SQLite.SQLiteCommand(
@@ -2702,6 +2708,15 @@ namespace URLNotesGrabberCORE
 
         private static string FormatKeyLabel(ApiKeyConfig key) => $"[Key#{key.KeyNumber}]";
 
+        private static string SummarizeBody(string body)
+        {
+            if (string.IsNullOrWhiteSpace(body))
+                return "(empty)";
+
+            var flat = System.Text.RegularExpressions.Regex.Replace(body, @"<[^>]+>|\s+", " ").Trim();
+            return flat.Length <= 80 ? flat : flat.Substring(0, 80) + "...";
+        }
+
         private static int GetRetryDelaySecondsFromHeaders(IEnumerable<HeaderParameter>? headers)
         {
             if (headers == null)
@@ -2780,11 +2795,27 @@ namespace URLNotesGrabberCORE
                 Console.WriteLine($"{FormatKeyLabel(key)} {timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
                 var myDeserializedClass = new Root();
 
+                // Never reached the API: there is no body to interpret, so the post's state is still unknown.
+                if (response.ResponseStatus != ResponseStatus.Completed)
+                {
+                    myDeserializedClass.statusCode = response.ResponseStatus.ToString();
+                    myDeserializedClass.transientFailure = true;
+                    Console.WriteLine($"[Transient] {FormatKeyLabel(key)} transport {response.ResponseStatus}: {response.ErrorException?.Message}");
+                    return myDeserializedClass;
+                }
+
                 try
                 {
                     var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
-                    if (deserializedResult != null)
+                    if (deserializedResult == null)
                     {
+                        // Empty body behind an HTTP status: an edge/proxy response, not the API.
+                        myDeserializedClass.statusCode = response.StatusCode.ToString();
+                        myDeserializedClass.transientFailure = true;
+                        Console.WriteLine($"[Transient] {FormatKeyLabel(key)} HTTP {(int)response.StatusCode} {response.StatusDescription} — empty body");
+                        return myDeserializedClass;
+                    }
+
                     myDeserializedClass = deserializedResult;
                     myDeserializedClass.rawJson = myJsonResponse;
 
@@ -2796,128 +2827,31 @@ namespace URLNotesGrabberCORE
                     bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
                     bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
 
-                        if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
+                    if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
                     {
-                            if (response?.Headers != null)
-                            {
-                                bool checkResetLocal = false;
-                                foreach (var header in response.Headers)
-                                {
-                                    string? headerName = header?.Name;
-                                    string? headerValue = header?.Value?.ToString();
-                                    if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
-                                        continue;
-
-                                    if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
-                                    {
-                                        if (int.TryParse(headerValue, out int retrySecs))
-                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
-                                        else if (DateTimeOffset.TryParse(headerValue, out var dto))
-                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
-                                    }
-
-                                    if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
-                                    {
-                                        var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
-                                        myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
-                                    }
-
-                                    if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
-                                        checkResetLocal = true;
-
-                                    if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
-                                    {
-                                        if (int.TryParse(headerValue, out int resetValue))
-                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
-                                        else if (long.TryParse(headerValue, out long epochVal))
-                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
-                                    }
-                                }
-                            }
-
+                        myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, GetRetryDelaySecondsFromHeaders(response.Headers));
                         myDeserializedClass.statusCode = "TooManyRequests";
                     }
                 }
-                }
                 catch (Exception ex)
                 {
-                    Console.WriteLine($"Failed JSON: {myJsonResponse}");
-                    Console.WriteLine(ex.ToString());
+                    // A body that will not parse came from infrastructure (CDN/proxy/WAF), not the Tumblr
+                    // API, so it says nothing about this post. Retryable, not a failure of the post itself.
+                    myDeserializedClass.statusCode = response.StatusCode.ToString();
 
-                    if (!response.IsSuccessful)
-                    {
-                        string? statusStr = null;
-                        try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
-                        Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
-                        if (!string.IsNullOrEmpty(statusStr))
-                            myDeserializedClass.statusCode = statusStr;
-
-                        bool checkReset = false;
-
-                        if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
-                        {
-                            bool foundRateLimitHeader = false;
-                            foreach (var header in response.Headers)
-                            {
-                                if (header.Name != null && header.Value != null)
-                                {
-                                    Console.WriteLine($"{header.Name}  -   {header.Value}");
-                                    var headerValue = header.Value?.ToString();
-                                    if (!string.IsNullOrEmpty(headerValue))
-                                    {
-                                        if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
-                                        {
-                                            if (int.TryParse(headerValue, out int retrySecs))
-                                            {
-                                                if (myDeserializedClass.retryInSeconds < retrySecs)
-                                                    myDeserializedClass.retryInSeconds = retrySecs;
-                                            }
-                                            else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
-                                            {
-                                                var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
-                                                if (myDeserializedClass.retryInSeconds < secs)
-                                                    myDeserializedClass.retryInSeconds = secs;
-                                            }
-                                        }
-
-                                        if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
-                                        {
-                                            if (int.TryParse(headerValue, out int resetValue))
-                                            {
-                                                if (myDeserializedClass.retryInSeconds < resetValue)
-                                                    myDeserializedClass.retryInSeconds = resetValue;
-                                            }
-                                            else if (long.TryParse(headerValue, out long epochVal))
-                                            {
-                                                var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
-                                                if (myDeserializedClass.retryInSeconds < secs)
-                                                    myDeserializedClass.retryInSeconds = secs;
-                                            }
-                                        }
-
-                                        if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
-                                        {
-                                            if (long.TryParse(headerValue, out long epoch))
-                                            {
-                                                var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
-                                                if (myDeserializedClass.retryInSeconds < secs)
-                                                    myDeserializedClass.retryInSeconds = secs;
-                                            }
-                                            foundRateLimitHeader = true;
-                                        }
-                                    }
-                                    if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
-                                        checkReset = true;
-                                    else
-                                        checkReset = false;
-                                }
-
-                                if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
+                    if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
                     {
+                        myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
                         myDeserializedClass.statusCode = "TooManyRequests";
                     }
-                            }
-                        }
+                    else
+                    {
+                        myDeserializedClass.transientFailure = true;
+                        Console.WriteLine($"[Transient] {FormatKeyLabel(key)} HTTP {(int)response.StatusCode} {response.StatusDescription} — unparseable body: {SummarizeBody(myJsonResponse)}");
+
+                        // A 2xx that will not parse is a genuine surprise; keep the detail for that case only.
+                        if (response.IsSuccessful)
+                            Console.WriteLine(ex.ToString());
                     }
                 }
 

URLNotesGrabberCORE/Program.cs

@@ -281,7 +281,7 @@ namespace URLNotesGrabberCORE
                             managedCollectRun = true;
                         }
 
-                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
+                        exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
                         break;
 
                     case "--blogsR": //collect notes from all posts
@@ -452,7 +452,7 @@ namespace URLNotesGrabberCORE
             Console.WriteLine("--importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");
 
             Console.WriteLine();
-            Console.WriteLine("Exit status: 0 = success; 1 = unexpected error; 2 = usage error (unknown command or bad/missing arguments)");
+            Console.WriteLine("Exit status: 0 = success; 1 = unexpected error; 2 = usage error (unknown command or bad/missing arguments); 3 = incomplete (--collect paused on a rate limit, or skipped posts after transient API failures) - relaunch to resume");
         }
 
         static void WritePostBlogsToFile(string outPath)
@@ -582,16 +582,11 @@ namespace URLNotesGrabberCORE
 
         protected static string NormalizeBlogFolderName(string folderName)
         {
-            return folderName
-                .Replace("_1", "")
-                .Replace("_2", "")
-                .Replace("_3", "")
-                .Replace("_4", "")
-                .Replace("_5", "")
-                .Replace("_6", "")
-                .Replace("_7", "")
-                .Replace("_8", "")
-                .Replace("_9", "");
+            // Archive tools suffix duplicate blog folders with _1, _2, ... _10 and beyond. Strip only a
+            // trailing numeric suffix: unanchored substring removal ate the "_1" inside "_10" and left the
+            // "0" welded to the name (zomb-eh_10 -> zomb-eh0), and mangled any blog whose real name
+            // contains "_1". A blog name is never a prefix of itself plus "_<digits>", so this is safe.
+            return System.Text.RegularExpressions.Regex.Replace(folderName, @"_\d+$", "");
         }
 
         static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
@@ -847,7 +842,9 @@ namespace URLNotesGrabberCORE
 
                 RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
                 {
-                    PermitLimit = 300,
+                    // 1/sec average, matching --collect: the CDN reacts to aggregate traffic from the IP,
+                    // not to per-command rates.
+                    PermitLimit = 60,
                     QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                     QueueLimit = 1,
                     Window = TimeSpan.FromMinutes(1),
@@ -883,7 +880,9 @@ namespace URLNotesGrabberCORE
 
                     while (hasMoreLikes)
                     {
-                        using RateLimitLease lease = limiter.AttemptAcquire(1);
+                        // Wait for a permit rather than giving up on one: the limiter paces the loop, it is
+                        // not a failure condition. Only one acquire is ever pending, so QueueLimit = 1 suffices.
+                        using RateLimitLease lease = await limiter.AcquireAsync(1);
                         if (!lease.IsAcquired)
                         {
                             Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
@@ -1129,6 +1128,43 @@ if (shouldInsert)
             }
         }
 
+        // Backoff between in-place retries of a transient infrastructure failure. Most CDN 403s and edge
+        // 5xxs clear within a few seconds, so retrying here saves the post its single attempt for the pass.
+        static readonly int[] TransientBackoffSeconds = { 1, 4, 10 };
+
+        // Fetches one page, retrying transient failures in place. Rate limits are returned to the caller
+        // untouched — those are handled by pausing the whole run, not by retrying this post.
+        static async Task<Root> FetchNotesPage(Tuple<string, long, long, long> post, string beforeTimestamp)
+        {
+            Root response = null!;
+
+            for (int attempt = 0; ; attempt++)
+            {
+                var key = ApiKeyPool.GetCurrentKey();
+                response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
+
+                if (response.statusCode == "TooManyRequests")
+                {
+                    ApiKeyPool.MarkRateLimited(key, response.retryInSeconds > 0 ? response.retryInSeconds : 60);
+                    return response;
+                }
+
+                if (!response.transientFailure)
+                {
+                    // Only a response that actually reached the API says anything about the key's standing.
+                    ApiKeyPool.MarkAvailable(key);
+                    return response;
+                }
+
+                if (attempt >= TransientBackoffSeconds.Length)
+                    return response;
+
+                int delay = TransientBackoffSeconds[attempt];
+                Console.WriteLine($"[Transient] retry {attempt + 1}/{TransientBackoffSeconds.Length} in {delay}s");
+                await Task.Delay(delay * 1000);
+            }
+        }
+
         static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
         {
             try
@@ -1146,18 +1182,16 @@ if (shouldInsert)
                 string beforeTimestamp = post.Item3.ToString();
                 bool hasReplies = false;
                 const int maxPages = 500;
-                var key = ApiKeyPool.GetCurrentKey();
-                var response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
+                var response = await FetchNotesPage(post, beforeTimestamp);
 
                 if (response.statusCode == "TooManyRequests")
-                {
-                    int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
-                    ApiKeyPool.MarkRateLimited(key, retry);
                     return "TooManyRequests";
-                }
 
-                if (response.meta?.status != 429)
-                    ApiKeyPool.MarkAvailable(key);
+                if (response.transientFailure)
+                {
+                    Console.WriteLine($"[Skip] {post.Item1}/{post.Item2} — {response.statusCode} after {TransientBackoffSeconds.Length} retries");
+                    return "Transient";
+                }
 
                 if (IsNotFound(response))
                 {
@@ -1166,19 +1200,6 @@ if (shouldInsert)
                     Thread.Sleep(1000);
                     return "NotFound";
                 }
-                if (response == null)
-                {
-                    Console.WriteLine("##### Response is null - API Failure? ###");
-                    return "FAILURE";
-                }
-                if (response.statusCode == "TooManyRequests")
-                {
-                    int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
-                    ApiKeyPool.MarkRateLimited(key, retry);
-
-                    ApiKeyPool.SleepUntilAnyAvailable(30);
-                    return response.statusCode;
-                }
 
                 // Pagination loop
                 while (true)
@@ -1228,16 +1249,15 @@ if (shouldInsert)
                         break;
                     }
 
-                    key = ApiKeyPool.GetCurrentKey();
-                    response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
+                    response = await FetchNotesPage(post, beforeTimestamp);
                     if (response.statusCode == "TooManyRequests")
-                    {
-                        int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
-                        ApiKeyPool.MarkRateLimited(key, retry);
                         return "TooManyRequests";
+
+                    if (response.transientFailure)
+                    {
+                        Console.WriteLine($"[Skip] {post.Item1}/{post.Item2} — {response.statusCode} on page {page} after {TransientBackoffSeconds.Length} retries");
+                        return "Transient";
                     }
-                    if (response.meta?.status != 429)
-                        ApiKeyPool.MarkAvailable(key);
 
                     if (IsNotFound(response))
                     {
@@ -1268,7 +1288,11 @@ if (shouldInsert)
             return "UNKNOWN";
         }
 
-        static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
+        // Consecutive transient failures that mean the API edge is rejecting traffic wholesale rather than
+        // blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
+        const int MaxConsecutiveTransient = 10;
+
+        static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
         {
             List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
 
@@ -1277,9 +1301,14 @@ if (shouldInsert)
             // post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
             HashSet<(string, long)> attempted = new HashSet<(string, long)>();
 
+            int skipped = 0;
+            int consecutiveTransient = 0;
+
             RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
             {
-                PermitLimit = 300,
+                // 1/sec average. Sustained higher rates draw CDN-level 403s that the API's own rate-limit
+                // headers never warn about, so this sits well under the per-key quota on purpose.
+                PermitLimit = 60,
                 QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                 QueueLimit = 1,
                 Window = TimeSpan.FromMinutes(1),
@@ -1304,19 +1333,35 @@ if (shouldInsert)
 
                         ApiKeyPool.SleepUntilAnyAvailable(30);
 
-                        string status;
-
-                        using RateLimitLease lease = limiter.AttemptAcquire(1);
-                        if (lease.IsAcquired)
+                        // Wait for a permit rather than giving up on one: the limiter paces the loop, it is
+                        // not a failure condition. Only one acquire is ever pending, so QueueLimit = 1 suffices.
+                        using RateLimitLease lease = await limiter.AcquireAsync(1);
+                        if (!lease.IsAcquired)
                         {
+                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
+                            return 3; // abort without completing the run so a later launch resumes
+                        }
+
                         Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
-                            status = await GrabNotes(post);
+                        string status = await GrabNotes(post);
+
+                        if (status == "Transient")
+                        {
+                            // Retries in GrabNotes are already exhausted. Skip the post so the pass can make
+                            // progress; it stays unmarked in the DB, so the next launch picks it up again.
+                            attempted.Add((post.Item1, post.Item2));
+                            skipped++;
+                            consecutiveTransient++;
+
+                            if (consecutiveTransient >= MaxConsecutiveTransient)
+                            {
+                                Console.WriteLine($"[Abort] {consecutiveTransient} consecutive transient failures - the API edge is rejecting traffic. Pausing run; relaunch to resume. ({skipped} post(s) skipped)");
+                                return 3;
+                            }
                         }
                         else
                         {
-                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
-                            return; // throttle: abort without completing the run so a later launch resumes
-                        }
+                            consecutiveTransient = 0;
 
                             if (status == "Success")
                             {
@@ -1333,8 +1378,8 @@ if (shouldInsert)
                             {
                                 // Throttle, not a real per-post failure: don't consume this post's single attempt.
                                 // Abort the pass without completing so a later launch resumes against the same cutoff.
-                            Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
-                            return;
+                                Console.WriteLine($"GrabNotes Result: TooManyRequests - pausing run; relaunch to resume. ({skipped} post(s) skipped)");
+                                return 3;
                             }
                             else
                             {
@@ -1342,6 +1387,7 @@ if (shouldInsert)
                                 attempted.Add((post.Item1, post.Item2));
                                 Console.WriteLine("GrabNotes Result: " + status);
                             }
+                        }
 
                         // Re-fetch the updated list after processing the current post
                         posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
@@ -1354,14 +1400,42 @@ if (shouldInsert)
                     DataAccess.CompleteCollectRun();
                     Console.WriteLine("Full re-check run complete.");
                 }
+
+                if (skipped > 0)
+                {
+                    Console.WriteLine($"Pass finished with {skipped} post(s) skipped after transient failures; relaunch to retry them.");
+                    return 3;
+                }
+
+                return 0;
             }
             catch (Exception ex)
             {
                 Console.WriteLine(ex.ToString());
+                return 1;
             }
         }
 
 
+        // Field prefixes TraverseDirectory recognizes as the start of a new record field.
+        // Used to know where a multi-line Body/Downloaded files value ends.
+        private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
+        {
+            "Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
+            "Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
+            "Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
+            "Tags:", "Title:"
+        };
+
+        private static bool IsTraverseDirectoryFieldLine(string line)
+        {
+            foreach (var prefix in TraverseDirectoryFieldPrefixes)
+            {
+                if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
+            }
+            return false;
+        }
+
         static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
         {
             DateTime directoryStart = DateTime.Now;
@@ -1398,8 +1472,10 @@ if (shouldInsert)
                                 var urls = new List<string>();
                                 var reblog = new ReblogRecord();
 
-                                foreach (string line in File.ReadLines(file))
+                                string[] fileLines = File.ReadAllLines(file);
+                                for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
                                 {
+                                    string line = fileLines[lineIndex];
                                     if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
                                     {
                                         if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
@@ -1420,7 +1496,7 @@ if (shouldInsert)
                                                 DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                     reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                     reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
-                                                                    reblog.title, false);
+                                                                    reblog.title, false, rootURL: reblog.rootURL);
                                                 recordImportStopwatch.Stop();
 
                                                 postsAdded++;
@@ -1445,9 +1521,21 @@ if (shouldInsert)
                                     {
                                         reblog.reblogName = line.Substring(13).Trim();
                                     }
+                                    if (line.StartsWith(@"Reblog root url:", StringComparison.OrdinalIgnoreCase))
+                                    {
+                                        reblog.rootURL = line.Substring(16).Trim();
+                                    }
                                     if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
                                     {
-                                        reblog.downloadedFiles = line.Substring(17).Trim();
+                                        var valueLines = new List<string> { line.Substring(17).Trim() };
+                                        int nextLineIndex = lineIndex + 1;
+                                        while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
+                                        {
+                                            valueLines.Add(fileLines[nextLineIndex]);
+                                            nextLineIndex++;
+                                        }
+                                        reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
+                                        lineIndex = nextLineIndex - 1;
                                     }
                                     if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
                                     {
@@ -1459,7 +1547,15 @@ if (shouldInsert)
                                     }
                                     if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
                                     {
-                                        reblog.body = line.Substring(6).Trim();
+                                        var valueLines = new List<string> { line.Substring(6).Trim() };
+                                        int nextLineIndex = lineIndex + 1;
+                                        while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
+                                        {
+                                            valueLines.Add(fileLines[nextLineIndex]);
+                                            nextLineIndex++;
+                                        }
+                                        reblog.body = string.Join("\n", valueLines).Trim();
+                                        lineIndex = nextLineIndex - 1;
                                     }
                                     if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
                                     {
@@ -1548,7 +1644,7 @@ if (shouldInsert)
                                             DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                     reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                     reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
-                                                                    reblog.title, true);
+                                                                    reblog.title, true, rootURL: reblog.rootURL);
                                             recordImportStopwatch.Stop();
 
                                             postsAdded++;

URLNotesGrabberCORE/ResponseNotes.cs

@@ -85,6 +85,10 @@ namespace URLNotesGrabberCORE
         public int retryInSeconds { get; set; }
 
         public string rawJson { get; set; }
+
+        // The request never reached the Tumblr API (transport error, or an edge/CDN response with a
+        // non-JSON body). Says nothing about the post, so the caller should retry rather than fail it.
+        public bool transientFailure { get; set; }
     }
 
     // Classes for Posts API endpoint (for reply_text)