sleep call on finding note

Merge branch 'claude/goofy-antonelli-73ea26'
collapse GetPosts SQL log to a single line
2026-05-15 11:49:54 -05:00 · 2026-05-11 09:09:25 -05:00 · 2026-05-11 09:09:24 -05:00 · 2026-05-11 09:03:12 -05:00 · 2026-05-11 09:03:01 -05:00 · 2026-05-11 08:37:47 -05:00
3 changed files with 203 additions and 156 deletions
@@ -362,3 +362,12 @@ MigrationBackup/
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
 /URLNotesGrabberCORE/TL.db
+
+# Claude Code local settings + worktrees
+.claude/settings.local.json
+.claude/worktrees/
+
+/URLNotesGrabberCORE/TL.db
+/URLNotesGrabberCORE/tl.db-shm
+/URLNotesGrabberCORE/tl.db-wal
+/U/jim/documents/web copies/blogs
@@ -526,7 +526,7 @@ namespace URLNotesGrabberCORE
                        Console.ForegroundColor = ConsoleColor.Green;
                        Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
                        Console.ForegroundColor = previousColor;
-                        //Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
+                        Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
                    }
                    else
                    {
@@ -665,7 +665,7 @@ namespace URLNotesGrabberCORE
                    if (beforeDate.HasValue)
                    {
                        long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
-                        sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
+                        sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime= 0) " + Environment.NewLine;
                    }

                    sql += "GROUP BY " + Environment.NewLine +
@@ -675,7 +675,7 @@ namespace URLNotesGrabberCORE
                }

                Console.WriteLine(withoutNotesOnly);
-                Console.WriteLine(sql);
+                Console.WriteLine(System.Text.RegularExpressions.Regex.Replace(sql, @"\s+", " ").Trim());
                Console.Write(">" ); //Console.ReadKey();
                //Thread.Sleep(250);

@@ -972,7 +972,7 @@ namespace URLNotesGrabberCORE
                if (!string.IsNullOrEmpty(specificBlog))
                    sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
                else
-                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName AND EXISTS (SELECT 1 FROM Posts P WHERE P.BlogName = B.BlogName) GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
+                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";

                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                {
@@ -1177,7 +1177,7 @@ namespace URLNotesGrabberCORE
            try
            {
                // Informative output when marking a post as NotFound
-                Console.WriteLine($"Marking post NotFound: {blogName}/{postID}");
+                Console.WriteLine($"Marking post NotFound: {blogName}.tumblr.com/post/{postID}");

                connection.Open();

@@ -1467,36 +1467,43 @@ namespace URLNotesGrabberCORE
            return APICount;
        }

-        public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
+        public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
        {
            DBPath ??= GetDefaultDbPath();
            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            int rowsAffected = 0;
+
+            // A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
+            if (replyText == ".")
+                replyText = ". ";

            try
            {
                connection.Open();

                //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
-                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
+                // Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
+                // Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text.
+                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.')";
                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                {
-                    command.Parameters.AddWithValue("@replyText", replyText ?? ".");
+                    command.Parameters.AddWithValue("@replyText", replyText ?? "?");
                    command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                    command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
                    command.Parameters.AddWithValue("@PostID", postID);
                    command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
                    command.Parameters.AddWithValue("@TimeStamp", timestamp);
-                    int rowsAffected = command.ExecuteNonQuery();
+                    rowsAffected = command.ExecuteNonQuery();

                    if (rowsAffected == 0)
                    {
-                        Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
+                        Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}.tumblr.com/post/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
                        Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
                        Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
                    }
                    else
                    {
-                        Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}");
+                        Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}.tumblr.com/post/{postID} from {noteBlogName}");
                    }
                }
            }
@@ -1510,9 +1517,10 @@ namespace URLNotesGrabberCORE
            {
                connection.Close();
            }
+            return rowsAffected;
        }

-        public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
+        public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
        {
            DBPath ??= GetDefaultDbPath();
            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -1532,14 +1540,16 @@ namespace URLNotesGrabberCORE

                    if (rowsAffected == 0)
                    {
-                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}/{postID} (rows not found or values unchanged)");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}.tumblr.com/post/{postID} (rows not found or values unchanged)");
                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
                    }
                    else
                    {
-                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}.tumblr.com/post/{postID} with '{replyText}'");
                    }
+
+                    return rowsAffected;
                }
            }
            catch (Exception ex)
@@ -1547,6 +1557,7 @@ namespace URLNotesGrabberCORE
                // Breakpoint here
                Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
                Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
+                return 0;
            }
            finally
            {
@@ -1590,7 +1601,7 @@ namespace URLNotesGrabberCORE

        public static void Initialize(IConfiguration config, string? dbPath, string configFilePath, string? overrideSection = null)
        {
-            _dbPath = dbPath ?? "..\\..\\..\\tl.db";
+            _dbPath = dbPath ?? throw new ArgumentNullException(nameof(dbPath));
            _configFilePath = configFilePath;
            EnsureStateTableExists();

@@ -1937,6 +1948,28 @@ namespace URLNotesGrabberCORE
            return all;
        }

+        public static void SleepUntilAnyAvailable(int refreshSeconds = 30)
+        {
+            if (refreshSeconds <= 0) refreshSeconds = 30;
+            if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return;
+
+            DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
+            int remaining = minRetry;
+            while (remaining > 0)
+            {
+                Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T"));
+                int sleepSeconds = Math.Min(refreshSeconds, remaining);
+                Thread.Sleep(sleepSeconds * 1000);
+                remaining -= sleepSeconds;
+
+                if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining)
+                {
+                    remaining = refreshed;
+                    retryAt = DateTime.Now.AddSeconds(remaining);
+                }
+            }
+        }
+
        private static void SaveState()
        {
            using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
@@ -85,8 +85,8 @@ namespace URLNotesGrabberCORE
            args = filteredArgs.ToArray();

            string? dbPath = config["appSettings:PathDB"];
-            if (string.IsNullOrWhiteSpace(dbPath))
-                dbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "tl.db");
+            if (string.IsNullOrEmpty(dbPath))
+                throw new InvalidOperationException("PathDB is not configured in appsettings.json");

            ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null);

@@ -244,7 +244,7 @@ namespace URLNotesGrabberCORE
                            }
                        }

-                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
+                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
                        break;

                    case "-blogsR": //collect notes from all posts
@@ -373,7 +373,7 @@ namespace URLNotesGrabberCORE
            {
                foreach (var post in posts)
                {
-                    Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
+                    Console.WriteLine(@"https://{0}.tumblr.com/post/{1}", post.Item1, post.Item2);
                    sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
                }
            }
@@ -449,12 +449,24 @@ namespace URLNotesGrabberCORE

        static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
        {
+            const int MaxPages = 10;
            try
            {
-                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
+                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}.tumblr.com/post/{postID}");
+
+                int replyCount = 0;
+                int rowsUpdated = 0;
+                int emptyReplyCount = 0;
+                long pageTimestamp = 0;
+                int page = 0;
+                bool sawHealthyResponse = false;
+
+                while (page < MaxPages)
+                {
+                    page++;
                    await Task.Delay(2000);
                    var key = ApiKeyPool.GetCurrentKey();
-                var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp);
+                    var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);

                    if (postsResponse?.statusCode == "TooManyRequests")
                    {
@@ -467,38 +479,48 @@ namespace URLNotesGrabberCORE
                    if (postsResponse?.meta?.status != 429)
                        ApiKeyPool.MarkAvailable(key);

+                    if (postsResponse?.meta?.status == 200)
+                        sawHealthyResponse = true;
+
                    if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
                    {
-                    Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}");
-                    Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
-                    Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
-                    if (postsResponse?.response != null)
-                    {
-                        Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
-                    }
-                    
-                    var previousColor = Console.ForegroundColor;
+                        var prevColor = Console.ForegroundColor;
                        Console.ForegroundColor = ConsoleColor.Yellow;
-                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response");
-                    Console.ForegroundColor = previousColor;
-                    DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                        Console.WriteLine($"[Reply Text] No notes returned for {blogName}.tumblr.com/post/{postID} on page {page} (before_timestamp={pageTimestamp})");
+                        Console.ForegroundColor = prevColor;
+                        Console.WriteLine($"[Reply Text]   meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
+                        var raw = postsResponse?.rawJson ?? string.Empty;
+                        if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
+                        Console.WriteLine($"[Reply Text]   raw: {raw}");

-                    return;
+                        bool isNotFound = postsResponse?.meta?.status == 404
+                            || string.Equals(postsResponse?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
+                        if (isNotFound)
+                        {
+                            var notFoundColor = Console.ForegroundColor;
+                            Console.ForegroundColor = ConsoleColor.Yellow;
+                            Console.WriteLine($"[Reply Text] Post {blogName}.tumblr.com/post/{postID} returned 404 - marking NotFound=1 and replies '?'");
+                            Console.ForegroundColor = notFoundColor;
+                            DataAccess.UpdatePostMarkNotFound(blogName, postID);
+                            rowsUpdated += DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                            emptyReplyCount += rowsUpdated;
+                        }
+                        break;
                    }

-                // Update each reply with its text
-                int replyCount = 0;
-                int skippedCount = 0;
+                    long lastNoteTimestamp = 0;
                    foreach (var note in postsResponse.response.notes)
                    {
+                        if (note.timestamp > 0)
+                            lastNoteTimestamp = note.timestamp;
+
                        if (note.type == "reply")
                        {
                            if (!string.IsNullOrEmpty(note.reply_text))
                            {
-                            DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
+                                rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                                replyCount++;

-                            // Output the reply text being stored
                                string displayText = note.reply_text.Length > 100
                                    ? note.reply_text.Substring(0, 100) + "..."
                                    : note.reply_text;
@@ -509,17 +531,16 @@ namespace URLNotesGrabberCORE
                            }
                            else
                            {
-                            skippedCount++;
-                            Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
+                                rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
+                                emptyReplyCount++;
+                                Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
                            }
                        }
                        else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
                        {
-                        // Handle reblogs with comment
-                        DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
+                            rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                            replyCount++;

-                        // Output the reblog comment being stored
                            string displayText = note.reply_text.Length > 100
                                ? note.reply_text.Substring(0, 100) + "..."
                                : note.reply_text;
@@ -530,26 +551,30 @@ namespace URLNotesGrabberCORE
                        }
                    }

-                if (replyCount > 0)
+                    if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
                    {
-                    Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
+                        break;
                    }
-                else
-                {
-                    Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
+                    pageTimestamp = lastNoteTimestamp;
                }

-                // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
-                Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
-                //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
-                //if (cleanupCount > 0)
-                //{
-                //    Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
-                //}
+                // If pagination finished without updating any reply rows but the API responded healthily
+                // at least once, mark the post's outstanding '.' replies '?' so the work queue releases it.
+                if (rowsUpdated == 0 && sawHealthyResponse)
+                {
+                    var prevColor = Console.ForegroundColor;
+                    Console.ForegroundColor = ConsoleColor.Yellow;
+                    Console.WriteLine($"[Reply Text] Marking outstanding replies for {blogName}.tumblr.com/post/{postID} '?' (healthy 200 OK, no matching reply notes after {page} page(s))");
+                    Console.ForegroundColor = prevColor;
+                    rowsUpdated = DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                    emptyReplyCount += rowsUpdated;
+                }
+
+                Console.WriteLine($"[Reply Text] Done {blogName}.tumblr.com/post/{postID}: notes={replyCount}, rowsUpdated={rowsUpdated}, empty='?'={emptyReplyCount}, pages={page}");
            }
            catch (Exception ex)
            {
-                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}");
+                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}.tumblr.com/post/{postID}: {ex.Message}");
                Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
            }
        }
@@ -566,14 +591,16 @@ namespace URLNotesGrabberCORE
                Console.WriteLine();

                int totalProcessedCount = 0;
-                int totalReplies = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
+                int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;

-                Console.WriteLine($"[Reply Text] Total replies to process: {totalReplies}");
+                Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
                Console.WriteLine();

-                while (totalProcessedCount < totalReplies)
+                while (true)
                {
-                    var batch = DataAccess.GetRepliesWithFilledText();
+                    // Re-query each iteration so posts whose replies got filled in as a side-effect
+                    // of a previous post's update are skipped without burning an API call.
+                    var batch = DataAccess.GetRepliesWithFilledText(limit: 1);

                    if (batch is null || batch.Count == 0)
                    {
@@ -581,31 +608,31 @@ namespace URLNotesGrabberCORE
                        break;
                    }

-                    foreach (var reply in batch)
-                    {
-                        var blogName = (reply as Tuple<string, long, long>).Item1;
-                        var postID = (reply as Tuple<string, long, long>).Item2;
-                        var timestamp = (reply as Tuple<string, long, long>).Item3;
+                    var reply = batch[0];
+                    var blogName = reply.Item1;
+                    var postID = reply.Item2;
+                    var timestamp = reply.Item3;

                    await FetchAndStoreReplyText(blogName, postID, timestamp);
                    totalProcessedCount++;

-                        int remaining = totalReplies - totalProcessedCount;
-                        double completionPct = (totalProcessedCount / (double)totalReplies) * 100.0;
+                    int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
+                    double completionPct = initialTotal > 0
+                        ? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
+                        : 100.0;

                    var previousColor = Console.ForegroundColor;
                    Console.ForegroundColor = ConsoleColor.Cyan;
-                        Console.WriteLine($"[{remaining} remaining] [{totalProcessedCount}/{totalReplies}] ({completionPct:F1}%)");
+                    Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
                    Console.ForegroundColor = previousColor;

-                        if (totalProcessedCount % 50 == 0 && totalProcessedCount < totalReplies)
+                    if (totalProcessedCount % 50 == 0)
                    {
                        await Task.Delay(2000);
                    }
                }
-                }

-                Console.WriteLine($"[Reply Text] Complete. Total processed: {totalProcessedCount} replies.");
+                Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
            }
            catch (Exception ex)
            {
@@ -709,19 +736,7 @@ namespace URLNotesGrabberCORE
                            int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                            ApiKeyPool.MarkRateLimited(key, retry);

-                            if (ApiKeyPool.IsAllRateLimited(out int minRetry))
-                            {
-                                Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before retry");
-                                int remaining = minRetry;
-                                DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
-                                while (remaining > 0)
-                                {
-                                    Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
-                                    int sleepSeconds = Math.Min(60, remaining);
-                                    Thread.Sleep(sleepSeconds * 1000);
-                                    remaining -= sleepSeconds;
-                                }
-                            }
+                            ApiKeyPool.SleepUntilAnyAvailable(30);
                            continue;
                        }

@@ -927,7 +942,7 @@ namespace URLNotesGrabberCORE

                if (IsNotFound(response))
                {
-                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}");
+                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2}");
                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                    Thread.Sleep(1000);
                    return "NotFound";
@@ -942,19 +957,7 @@ namespace URLNotesGrabberCORE
                    int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                    ApiKeyPool.MarkRateLimited(key, retry);

-                    if (ApiKeyPool.IsAllRateLimited(out int minRetry))
-                    {
-                        Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before returning");
-                        int remaining = minRetry;
-                        DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
-                        while (remaining > 0)
-                        {
-                            Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
-                            int sleepSeconds = Math.Min(60, remaining);
-                            Thread.Sleep(sleepSeconds * 1000);
-                            remaining -= sleepSeconds;
-                        }
-                    }
+                    ApiKeyPool.SleepUntilAnyAvailable(30);
                    return response.statusCode;
                }

@@ -968,7 +971,7 @@ namespace URLNotesGrabberCORE
                    {
                        if (IsNotFound(response))
                        {
-                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (empty response payload)");
+                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (empty response payload)");
                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                            return "NotFound";
                        }
@@ -979,7 +982,7 @@ namespace URLNotesGrabberCORE
                    {
                        if (IsNotFound(response))
                        {
-                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (notes payload missing)");
+                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (notes payload missing)");
                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                            return "NotFound";
                        }
@@ -1002,7 +1005,7 @@ namespace URLNotesGrabberCORE
                    page++;
                    if (page > maxPages)
                    {
-                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination.");
+                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}.tumblr.com/post/{post.Item2}. Aborting further pagination.");
                        break;
                    }

@@ -1019,7 +1022,7 @@ namespace URLNotesGrabberCORE

                    if (IsNotFound(response))
                    {
-                        Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}");
+                        Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}.tumblr.com/post/{post.Item2}");
                        DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                        return "NotFound";
                    }
@@ -1046,7 +1049,7 @@ namespace URLNotesGrabberCORE
            return "UNKNOWN";
        }

-        static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
+        static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
        {
            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);

@@ -1066,6 +1069,8 @@ namespace URLNotesGrabberCORE
                {
                    while (posts.Count > 0)
                    {
+                        ApiKeyPool.SleepUntilAnyAvailable(30);
+
                        var post = posts[0]; // Process the first post in the list
                        string status;
Author	SHA1	Message	Date
jim	69ce36a1f5	sleep call on finding note	2026-05-15 11:49:54 -05:00
jim	52293f2021	Merge branch 'claude/goofy-antonelli-73ea26'	2026-05-11 09:09:25 -05:00
jimandClaude Opus 4.7	2b86ce9119	collapse GetPosts SQL log to a single line Per-iteration call in CollectNotes was dumping ~30 lines of newline-padded SQL to the console. Replace with a whitespace-collapsed single-line print. Co-Authored-By: Claude Opus 4.7 <[email protected]>	2026-05-11 09:09:24 -05:00
jim	a5c118ce3b	Merge branch 'claude/goofy-antonelli-73ea26'	2026-05-11 09:03:12 -05:00
jimandClaude Opus 4.7	47497d02bf	tweak GetPosts beforeDate filter and broaden GetBlogsForLikes - GetPosts: match NotesGatheredDateTime = 0 instead of IS NULL for the beforeDate cutoff. - GetBlogsForLikes: drop the EXISTS-Posts predicate so blogs with notes but no posts rows are still eligible for likes collection. Co-Authored-By: Claude Opus 4.7 <[email protected]>	2026-05-11 09:03:01 -05:00
jimandClaude Opus 4.7	5ae1d4feb8	fix: skip API calls when all keys are rate-limited in -collect mode Pre-flight check on the CollectNotes loop now sleeps until at least one key recovers instead of issuing a wasted 429-bound request per iteration. Extracts the countdown into ApiKeyPool.SleepUntilAnyAvailable (30s refresh) and reuses it in CollectLikes and GrabNotes. Co-Authored-By: Claude Opus 4.7 <[email protected]>	2026-05-11 08:37:47 -05:00
jim	2034b43e83	Merge branch 'claude/wizardly-antonelli-9f2795'	2026-05-09 20:24:53 -05:00
jimandClaude Opus 4.7	4f00adcc05	fix: -collect mode now processes all posts instead of exiting after first Changed CollectNotes from async void to async Task and await it with .GetAwaiter().GetResult() to match the pattern used by -replies and -likes. Previously the method would return immediately after the first await, causing Main to exit before the loop could process more than one post. Co-Authored-By: Claude Opus 4.7 <[email protected]>	2026-05-09 20:24:50 -05:00
jim	bcbaea8c48	Merge branch 'claude/wizardly-antonelli-9f2795'	2026-05-08 21:42:39 -05:00
jimandClaude Opus 4.7	349b465f8a	fix: release 404 posts from -replies queue Tumblr API 404s were leaving Posts.NotFound=0 and Notes.replyText='.', so GetRepliesWithFilledText kept re-selecting the same dead post and the loop spun forever. Mark NotFound=1 and replies '?' on 404. Co-Authored-By: Claude Opus 4.7 <[email protected]>	2026-05-08 21:42:30 -05:00
jim	acc5e885a8	feat: require PathDB config in appsettings.json	2026-05-08 16:24:43 -05:00
jimandClaude Sonnet 4.6	b68a9833d8	chore: format remaining blog/post log output as <blog>.tumblr.com/post/<id> Apply the URL format to the remaining 6 console log lines that still referenced posts in <blog>/<id> form (404 logs, max-page-limit log, DumpReplies console output). Co-Authored-By: Claude Sonnet 4.6 <[email protected]>	2026-05-07 16:16:29 -05:00
jim	89f2addf01	Merge branch 'claude/awesome-edison-d12b61'	2026-05-07 16:13:30 -05:00
jimandClaude Sonnet 4.6	b963489741	chore: format blog/post log output as <blog>.tumblr.com/post/<id> Console-friendly URL format for log lines that reference a specific post — easier to copy/paste into a browser when investigating output. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>	2026-05-07 16:12:00 -05:00
jim	d18ab92019	Merge branch 'claude/awesome-edison-d12b61'	2026-05-07 15:58:13 -05:00
jimandClaude Sonnet 4.6	4737baa288	fix: keep '.' as needs-processing sentinel, normalize real-dot replies Previous fix removed '.' from the GetRepliesWithFilledText SELECT, which broke processing of legacy null-substitute rows that need to be re-fetched. Restored '.' in the SELECT. To avoid the infinite loop when an actual API reply is the literal string ".", normalize it to ". " (dot + trailing space) inside UpdateNoteReplyText so the stored value no longer matches the sentinel. Also extended the fan-out UPDATE guard to match the SELECT criteria (NULL / '' / '.') so legacy '.' rows in other reblog copies can be filled in too. The guard still refuses to overwrite '?' or already-fetched text. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>	2026-05-07 15:56:53 -05:00
jim	51c40fe2bd	Merge branch 'claude/awesome-edison-d12b61'	2026-05-07 15:49:40 -05:00
jimandClaude Sonnet 4.6	51e2a8a1e1	fix: stop infinite loop when reply text is a literal dot '.' was used as a sentinel for 'not yet fetched' in the SELECT query, but it is also valid reply text. This caused any post where a reply text was literally '.' to stay in the work queue forever. Also fixes the fan-out UPDATE guard: previously it used IFNULL(replyText, '.') <> @newValue, which would overwrite '?' (confirmed-empty) rows with '.' when processing a dot reply elsewhere in the reblog chain, pulling completed posts back into the queue and causing the remaining counter to increase. Changes: - Remove OR replyText = '.' from GetRepliesWithFilledText SELECT - Restrict UpdateNoteReplyText fan-out to NULL/'' rows only - Use '?' not '.' as null-coalesce fallback in UpdateNoteReplyText Co-Authored-By: Claude Sonnet 4.6 <[email protected]>	2026-05-07 15:49:35 -05:00
jim	0213478a40	Merge branch 'claude/epic-bardeen-cbbb7e' into master	2026-05-07 15:02:48 -05:00
jimandClaude Opus 4.7	081528a81a	fix: mark replies '?' after pagination yields no reply notes Previously, the '?' marker only fired when page 1 returned empty notes. Posts whose page 1 contained only likes/reblogs and whose page 2 came back empty were never marked, so GetRepliesWithFilledText kept reselecting them every iteration. Now mark after the pagination loop whenever rowsUpdated==0 and at least one page returned 200 OK. UpdateAllNoteReplyTextForPost now returns int so the caller can fold the count into emptyReplyCount/rowsUpdated for accurate Done logging. Co-Authored-By: Claude Opus 4.7 <[email protected]>	2026-05-07 15:01:04 -05:00
jim	f2539dc9d0	fix: mark post's replies '?' on confirmed empty notes response When the API responds 200 OK with notes:[] on the first page, the post has no conversational notes per Tumblr. Stamp all the post's reply rows '?' so the per-iteration requery stops handing this post back forever. Only fires for first-page 200 OK so a later-page empty (end of pagination) doesn't poison good rows.	2026-05-07 14:25:00 -05:00
jim	2438997c7c	chore: untrack and gitignore .claude local settings + worktrees Local Claude Code config; should not be in the repo.	2026-05-07 14:20:41 -05:00
jim	ac79727040	feat: fan reply updates across reblog chains, requery per post A reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it. Drop PostID from the UpdateNoteReplyText WHERE clause so a single API hit fills in the replyText on every matching row at once. Pair that with a per-iteration requery (limit 1) of the work list so posts whose replies were already filled in as a side-effect of a previous post's update are skipped without burning an API call.	2026-05-07 14:19:54 -05:00
jim	64a914c14d	fix: tolerate ~5s timestamp drift in UpdateNoteReplyText The API returns reply timestamps that are ~1s ahead of what -collect originally stored, so an exact TimeStamp match in the UPDATE was hitting zero rows for every note - the API call worked, the reply text came back, but nothing landed in the DB. Match within +-5s instead. Also return rowsAffected from UpdateNoteReplyText and report it separately from notes-seen in the summary so misses are visible.	2026-05-07 14:10:37 -05:00
jim	1c7b6a1e86	fix: start replies fetch from newest, ignore stale DB timestamp The DB-derived MAX(N.timestamp)+1 for a post is unreliable - many notes were stored with the same timestamp (likely crawl time, not the actual note timestamp), so passing it as before_timestamp excluded all real replies and returned an empty notes array. Diagnostics showed two unrelated posts coming back with identical totals (1625/1021/603) and zero notes. Now we start with no before_timestamp (newest page) and let pagination walk backward via each page's last-note timestamp.	2026-05-07 14:00:48 -05:00
jim	1da8a702ba	fix: pull reply_text from Tumblr in -replies mode Wrap FetchAndStoreReplyText in a per-post pagination loop (up to 10 pages, advancing before_timestamp via the last note's timestamp) so posts with >50 notes are fully walked. Log raw response (meta + first 500 chars of JSON) when a page returns no notes so empty results are diagnosable. Stop blanket-marking every reply on a post with '?' on the first empty response - rows stay '.' and remain retry-eligible; only individual replies that come back with empty reply_text are marked '?'.	2026-05-07 13:54:54 -05:00