From ac7972704057c3487f479ffaaeb220155f4db291 Mon Sep 17 00:00:00 2001 From: jim Date: Thu, 7 May 2026 14:19:54 -0500 Subject: [PATCH] feat: fan reply updates across reblog chains, requery per post A reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it. Drop PostID from the UpdateNoteReplyText WHERE clause so a single API hit fills in the replyText on every matching row at once. Pair that with a per-iteration requery (limit 1) of the work list so posts whose replies were already filled in as a side-effect of a previous post's update are skipped without burning an API call. --- .claude/settings.local.json | 7 +++++ URLNotesGrabberCORE/DataAccess.cs | 4 +-- URLNotesGrabberCORE/Program.cs | 52 ++++++++++++++++--------------- 3 files changed, 36 insertions(+), 27 deletions(-) create mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..61a5292 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(dotnet build *)" + ] + } +} diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 43398d4..2e0ef06 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -1478,8 +1478,8 @@ namespace URLNotesGrabberCORE connection.Open(); //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; - // Tolerance window on TimeStamp absorbs the ~1s drift between what -collect stored and what mode=conversation returns now. - string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText"; + // Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now. + string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@replyText", replyText ?? "."); diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index cc170be..d1d5cb8 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -562,14 +562,16 @@ namespace URLNotesGrabberCORE Console.WriteLine(); int totalProcessedCount = 0; - int totalReplies = DataAccess.GetRepliesWithFilledText()?.Count ?? 0; + int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0; - Console.WriteLine($"[Reply Text] Total replies to process: {totalReplies}"); + Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}"); Console.WriteLine(); - while (totalProcessedCount < totalReplies) + while (true) { - var batch = DataAccess.GetRepliesWithFilledText(); + // Re-query each iteration so posts whose replies got filled in as a side-effect + // of a previous post's update are skipped without burning an API call. + var batch = DataAccess.GetRepliesWithFilledText(limit: 1); if (batch is null || batch.Count == 0) { @@ -577,31 +579,31 @@ namespace URLNotesGrabberCORE break; } - foreach (var reply in batch) + var reply = batch[0]; + var blogName = reply.Item1; + var postID = reply.Item2; + var timestamp = reply.Item3; + + await FetchAndStoreReplyText(blogName, postID, timestamp); + totalProcessedCount++; + + int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0; + double completionPct = initialTotal > 0 + ? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0 + : 100.0; + + var previousColor = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Cyan; + Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)"); + Console.ForegroundColor = previousColor; + + if (totalProcessedCount % 50 == 0) { - var blogName = (reply as Tuple).Item1; - var postID = (reply as Tuple).Item2; - var timestamp = (reply as Tuple).Item3; - - await FetchAndStoreReplyText(blogName, postID, timestamp); - totalProcessedCount++; - - int remaining = totalReplies - totalProcessedCount; - double completionPct = (totalProcessedCount / (double)totalReplies) * 100.0; - - var previousColor = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine($"[{remaining} remaining] [{totalProcessedCount}/{totalReplies}] ({completionPct:F1}%)"); - Console.ForegroundColor = previousColor; - - if (totalProcessedCount % 50 == 0 && totalProcessedCount < totalReplies) - { - await Task.Delay(2000); - } + await Task.Delay(2000); } } - Console.WriteLine($"[Reply Text] Complete. Total processed: {totalProcessedCount} replies."); + Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}."); } catch (Exception ex) {