sleep call on finding note

Per-iteration call in CollectNotes was dumping ~30 lines of newline-padded
SQL to the console. Replace with a whitespace-collapsed single-line print.

Co-Authored-By: Claude Opus 4.7 <[email protected]>

.gitignore

@@ -362,3 +362,12 @@ MigrationBackup/
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
 /URLNotesGrabberCORE/TL.db
+
+# Claude Code local settings + worktrees
+.claude/settings.local.json
+.claude/worktrees/
+
+/URLNotesGrabberCORE/TL.db
+/URLNotesGrabberCORE/tl.db-shm
+/URLNotesGrabberCORE/tl.db-wal
+/U/jim/documents/web copies/blogs

URLNotesGrabberCORE/DataAccess.cs

@@ -526,7 +526,7 @@ namespace URLNotesGrabberCORE
                         Console.ForegroundColor = ConsoleColor.Green;
                         Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
                         Console.ForegroundColor = previousColor;
-                        //Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
+                        Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
                     }
                     else
                     {
@@ -665,7 +665,7 @@ namespace URLNotesGrabberCORE
                     if (beforeDate.HasValue)
                     {
                         long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
-                        sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
+                        sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime= 0) " + Environment.NewLine;
                     }
 
                     sql += "GROUP BY " + Environment.NewLine +
@@ -675,7 +675,7 @@ namespace URLNotesGrabberCORE
                 }
 
                 Console.WriteLine(withoutNotesOnly);
-                Console.WriteLine(sql);
+                Console.WriteLine(System.Text.RegularExpressions.Regex.Replace(sql, @"\s+", " ").Trim());
                 Console.Write(">" ); //Console.ReadKey();
                 //Thread.Sleep(250);
 
@@ -972,7 +972,7 @@ namespace URLNotesGrabberCORE
                 if (!string.IsNullOrEmpty(specificBlog))
                     sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
                 else
-                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName AND EXISTS (SELECT 1 FROM Posts P WHERE P.BlogName = B.BlogName) GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
+                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1177,7 +1177,7 @@ namespace URLNotesGrabberCORE
             try
             {
                 // Informative output when marking a post as NotFound
-                Console.WriteLine($"Marking post NotFound: {blogName}/{postID}");
+                Console.WriteLine($"Marking post NotFound: {blogName}.tumblr.com/post/{postID}");
 
                 connection.Open();
 
@@ -1467,36 +1467,43 @@ namespace URLNotesGrabberCORE
             return APICount;
         }
 
-        public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
+        public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            int rowsAffected = 0;
+
+            // A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
+            if (replyText == ".")
+                replyText = ". ";
 
             try
             {
                 connection.Open();
 
                 //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
-                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
+                // Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
+                // Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text.
+                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.')";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
-                    command.Parameters.AddWithValue("@replyText", replyText ?? ".");
+                    command.Parameters.AddWithValue("@replyText", replyText ?? "?");
                     command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                     command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
                     command.Parameters.AddWithValue("@PostID", postID);
                     command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
                     command.Parameters.AddWithValue("@TimeStamp", timestamp);
-                    int rowsAffected = command.ExecuteNonQuery();
+                    rowsAffected = command.ExecuteNonQuery();
 
                     if (rowsAffected == 0)
                     {
-                        Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
+                        Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}.tumblr.com/post/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
                         Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
                         Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
                     }
                     else
                     {
-                        Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}");
+                        Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}.tumblr.com/post/{postID} from {noteBlogName}");
                     }
                 }
             }
@@ -1510,9 +1517,10 @@ namespace URLNotesGrabberCORE
             {
                 connection.Close();
             }
+            return rowsAffected;
         }
 
-        public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
+        public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -1532,14 +1540,16 @@ namespace URLNotesGrabberCORE
 
                     if (rowsAffected == 0)
                     {
-                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}/{postID} (rows not found or values unchanged)");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}.tumblr.com/post/{postID} (rows not found or values unchanged)");
                         Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
                         Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
                     }
                     else
                     {
-                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}.tumblr.com/post/{postID} with '{replyText}'");
                     }
+
+                    return rowsAffected;
                 }
             }
             catch (Exception ex)
@@ -1547,6 +1557,7 @@ namespace URLNotesGrabberCORE
                 // Breakpoint here
                 Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
                 Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
+                return 0;
             }
             finally
             {
@@ -1590,7 +1601,7 @@ namespace URLNotesGrabberCORE
 
         public static void Initialize(IConfiguration config, string? dbPath, string configFilePath, string? overrideSection = null)
         {
-            _dbPath = dbPath ?? "..\\..\\..\\tl.db";
+            _dbPath = dbPath ?? throw new ArgumentNullException(nameof(dbPath));
             _configFilePath = configFilePath;
             EnsureStateTableExists();
 
@@ -1937,6 +1948,28 @@ namespace URLNotesGrabberCORE
             return all;
         }
 
+        public static void SleepUntilAnyAvailable(int refreshSeconds = 30)
+        {
+            if (refreshSeconds <= 0) refreshSeconds = 30;
+            if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return;
+
+            DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
+            int remaining = minRetry;
+            while (remaining > 0)
+            {
+                Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T"));
+                int sleepSeconds = Math.Min(refreshSeconds, remaining);
+                Thread.Sleep(sleepSeconds * 1000);
+                remaining -= sleepSeconds;
+
+                if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining)
+                {
+                    remaining = refreshed;
+                    retryAt = DateTime.Now.AddSeconds(remaining);
+                }
+            }
+        }
+
         private static void SaveState()
         {
             using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);

URLNotesGrabberCORE/Program.cs

@@ -85,8 +85,8 @@ namespace URLNotesGrabberCORE
             args = filteredArgs.ToArray();
 
             string? dbPath = config["appSettings:PathDB"];
-            if (string.IsNullOrWhiteSpace(dbPath))
+            if (string.IsNullOrEmpty(dbPath))
-                dbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "tl.db");
+                throw new InvalidOperationException("PathDB is not configured in appsettings.json");
 
             ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null);
 
@@ -244,7 +244,7 @@ namespace URLNotesGrabberCORE
                             }
                         }
 
-                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
+                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
                         break;
 
                     case "-blogsR": //collect notes from all posts
@@ -373,7 +373,7 @@ namespace URLNotesGrabberCORE
             {
                 foreach (var post in posts)
                 {
-                    Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
+                    Console.WriteLine(@"https://{0}.tumblr.com/post/{1}", post.Item1, post.Item2);
                     sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
                 }
             }
@@ -449,12 +449,24 @@ namespace URLNotesGrabberCORE
 
         static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
         {
+            const int MaxPages = 10;
             try
             {
-                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
+                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}.tumblr.com/post/{postID}");
+
+                int replyCount = 0;
+                int rowsUpdated = 0;
+                int emptyReplyCount = 0;
+                long pageTimestamp = 0;
+                int page = 0;
+                bool sawHealthyResponse = false;
+
+                while (page < MaxPages)
+                {
+                    page++;
                     await Task.Delay(2000);
                     var key = ApiKeyPool.GetCurrentKey();
-                var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp);
+                    var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);
 
                     if (postsResponse?.statusCode == "TooManyRequests")
                     {
@@ -467,38 +479,48 @@ namespace URLNotesGrabberCORE
                     if (postsResponse?.meta?.status != 429)
                         ApiKeyPool.MarkAvailable(key);
 
+                    if (postsResponse?.meta?.status == 200)
+                        sawHealthyResponse = true;
+
                     if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
                     {
-                    Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}");
+                        var prevColor = Console.ForegroundColor;
-                    Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
-                    Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
-                    if (postsResponse?.response != null)
-                    {
-                        Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
-                    }
-                    
-                    var previousColor = Console.ForegroundColor;
                         Console.ForegroundColor = ConsoleColor.Yellow;
-                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response");
+                        Console.WriteLine($"[Reply Text] No notes returned for {blogName}.tumblr.com/post/{postID} on page {page} (before_timestamp={pageTimestamp})");
-                    Console.ForegroundColor = previousColor;
+                        Console.ForegroundColor = prevColor;
-                    DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                        Console.WriteLine($"[Reply Text]   meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
+                        var raw = postsResponse?.rawJson ?? string.Empty;
+                        if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
+                        Console.WriteLine($"[Reply Text]   raw: {raw}");
 
-                    return;
+                        bool isNotFound = postsResponse?.meta?.status == 404
+                            || string.Equals(postsResponse?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
+                        if (isNotFound)
+                        {
+                            var notFoundColor = Console.ForegroundColor;
+                            Console.ForegroundColor = ConsoleColor.Yellow;
+                            Console.WriteLine($"[Reply Text] Post {blogName}.tumblr.com/post/{postID} returned 404 - marking NotFound=1 and replies '?'");
+                            Console.ForegroundColor = notFoundColor;
+                            DataAccess.UpdatePostMarkNotFound(blogName, postID);
+                            rowsUpdated += DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                            emptyReplyCount += rowsUpdated;
+                        }
+                        break;
                     }
 
-                // Update each reply with its text
+                    long lastNoteTimestamp = 0;
-                int replyCount = 0;
-                int skippedCount = 0;
                     foreach (var note in postsResponse.response.notes)
                     {
+                        if (note.timestamp > 0)
+                            lastNoteTimestamp = note.timestamp;
+
                         if (note.type == "reply")
                         {
                             if (!string.IsNullOrEmpty(note.reply_text))
                             {
-                            DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
+                                rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                                 replyCount++;
 
-                            // Output the reply text being stored
                                 string displayText = note.reply_text.Length > 100
                                     ? note.reply_text.Substring(0, 100) + "..."
                                     : note.reply_text;
@@ -509,17 +531,16 @@ namespace URLNotesGrabberCORE
                             }
                             else
                             {
-                            skippedCount++;
+                                rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
-                            Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
+                                emptyReplyCount++;
+                                Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
                             }
                         }
                         else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
                         {
-                        // Handle reblogs with comment
+                            rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
-                        DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                             replyCount++;
 
-                        // Output the reblog comment being stored
                             string displayText = note.reply_text.Length > 100
                                 ? note.reply_text.Substring(0, 100) + "..."
                                 : note.reply_text;
@@ -530,26 +551,30 @@ namespace URLNotesGrabberCORE
                         }
                     }
 
-                if (replyCount > 0)
+                    if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
                     {
-                    Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
+                        break;
                     }
-                else
+                    pageTimestamp = lastNoteTimestamp;
-                {
-                    Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
                 }
 
-                // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
+                // If pagination finished without updating any reply rows but the API responded healthily
-                Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
+                // at least once, mark the post's outstanding '.' replies '?' so the work queue releases it.
-                //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
+                if (rowsUpdated == 0 && sawHealthyResponse)
-                //if (cleanupCount > 0)
+                {
-                //{
+                    var prevColor = Console.ForegroundColor;
-                //    Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
+                    Console.ForegroundColor = ConsoleColor.Yellow;
-                //}
+                    Console.WriteLine($"[Reply Text] Marking outstanding replies for {blogName}.tumblr.com/post/{postID} '?' (healthy 200 OK, no matching reply notes after {page} page(s))");
+                    Console.ForegroundColor = prevColor;
+                    rowsUpdated = DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                    emptyReplyCount += rowsUpdated;
+                }
+
+                Console.WriteLine($"[Reply Text] Done {blogName}.tumblr.com/post/{postID}: notes={replyCount}, rowsUpdated={rowsUpdated}, empty='?'={emptyReplyCount}, pages={page}");
             }
             catch (Exception ex)
             {
-                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}");
+                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}.tumblr.com/post/{postID}: {ex.Message}");
                 Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
             }
         }
@@ -566,14 +591,16 @@ namespace URLNotesGrabberCORE
                 Console.WriteLine();
 
                 int totalProcessedCount = 0;
-                int totalReplies = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
+                int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
 
-                Console.WriteLine($"[Reply Text] Total replies to process: {totalReplies}");
+                Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
                 Console.WriteLine();
 
-                while (totalProcessedCount < totalReplies)
+                while (true)
                 {
-                    var batch = DataAccess.GetRepliesWithFilledText();
+                    // Re-query each iteration so posts whose replies got filled in as a side-effect
+                    // of a previous post's update are skipped without burning an API call.
+                    var batch = DataAccess.GetRepliesWithFilledText(limit: 1);
 
                     if (batch is null || batch.Count == 0)
                     {
@@ -581,31 +608,31 @@ namespace URLNotesGrabberCORE
                         break;
                     }
 
-                    foreach (var reply in batch)
+                    var reply = batch[0];
-                    {
+                    var blogName = reply.Item1;
-                        var blogName = (reply as Tuple<string, long, long>).Item1;
+                    var postID = reply.Item2;
-                        var postID = (reply as Tuple<string, long, long>).Item2;
+                    var timestamp = reply.Item3;
-                        var timestamp = (reply as Tuple<string, long, long>).Item3;
 
                     await FetchAndStoreReplyText(blogName, postID, timestamp);
                     totalProcessedCount++;
 
-                        int remaining = totalReplies - totalProcessedCount;
+                    int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
-                        double completionPct = (totalProcessedCount / (double)totalReplies) * 100.0;
+                    double completionPct = initialTotal > 0
+                        ? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
+                        : 100.0;
 
                     var previousColor = Console.ForegroundColor;
                     Console.ForegroundColor = ConsoleColor.Cyan;
-                        Console.WriteLine($"[{remaining} remaining] [{totalProcessedCount}/{totalReplies}] ({completionPct:F1}%)");
+                    Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
                     Console.ForegroundColor = previousColor;
 
-                        if (totalProcessedCount % 50 == 0 && totalProcessedCount < totalReplies)
+                    if (totalProcessedCount % 50 == 0)
                     {
                         await Task.Delay(2000);
                     }
                 }
-                }
 
-                Console.WriteLine($"[Reply Text] Complete. Total processed: {totalProcessedCount} replies.");
+                Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
             }
             catch (Exception ex)
             {
@@ -709,19 +736,7 @@ namespace URLNotesGrabberCORE
                             int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                             ApiKeyPool.MarkRateLimited(key, retry);
 
-                            if (ApiKeyPool.IsAllRateLimited(out int minRetry))
+                            ApiKeyPool.SleepUntilAnyAvailable(30);
-                            {
-                                Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before retry");
-                                int remaining = minRetry;
-                                DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
-                                while (remaining > 0)
-                                {
-                                    Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
-                                    int sleepSeconds = Math.Min(60, remaining);
-                                    Thread.Sleep(sleepSeconds * 1000);
-                                    remaining -= sleepSeconds;
-                                }
-                            }
                             continue;
                         }
 
@@ -927,7 +942,7 @@ namespace URLNotesGrabberCORE
 
                 if (IsNotFound(response))
                 {
-                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}");
+                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2}");
                     DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                     Thread.Sleep(1000);
                     return "NotFound";
@@ -942,19 +957,7 @@ namespace URLNotesGrabberCORE
                     int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                     ApiKeyPool.MarkRateLimited(key, retry);
 
-                    if (ApiKeyPool.IsAllRateLimited(out int minRetry))
+                    ApiKeyPool.SleepUntilAnyAvailable(30);
-                    {
-                        Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before returning");
-                        int remaining = minRetry;
-                        DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
-                        while (remaining > 0)
-                        {
-                            Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
-                            int sleepSeconds = Math.Min(60, remaining);
-                            Thread.Sleep(sleepSeconds * 1000);
-                            remaining -= sleepSeconds;
-                        }
-                    }
                     return response.statusCode;
                 }
 
@@ -968,7 +971,7 @@ namespace URLNotesGrabberCORE
                     {
                         if (IsNotFound(response))
                         {
-                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (empty response payload)");
+                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (empty response payload)");
                             DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                             return "NotFound";
                         }
@@ -979,7 +982,7 @@ namespace URLNotesGrabberCORE
                     {
                         if (IsNotFound(response))
                         {
-                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (notes payload missing)");
+                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (notes payload missing)");
                             DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                             return "NotFound";
                         }
@@ -1002,7 +1005,7 @@ namespace URLNotesGrabberCORE
                     page++;
                     if (page > maxPages)
                     {
-                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination.");
+                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}.tumblr.com/post/{post.Item2}. Aborting further pagination.");
                         break;
                     }
 
@@ -1019,7 +1022,7 @@ namespace URLNotesGrabberCORE
 
                     if (IsNotFound(response))
                     {
-                        Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}");
+                        Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}.tumblr.com/post/{post.Item2}");
                         DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                         return "NotFound";
                     }
@@ -1046,7 +1049,7 @@ namespace URLNotesGrabberCORE
             return "UNKNOWN";
         }
 
-        static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
+        static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
         {
             List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
 
@@ -1066,6 +1069,8 @@ namespace URLNotesGrabberCORE
                 {
                     while (posts.Count > 0)
                     {
+                        ApiKeyPool.SleepUntilAnyAvailable(30);
+
                         var post = posts[0]; // Process the first post in the list
                         string status;