jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: fan reply updates across reblog chains, requery per post

Browse Source 
A reply by a given blog at a given timestamp is the same reply
across the original post and every reblog of it. Drop PostID from
the UpdateNoteReplyText WHERE clause so a single API hit fills in
the replyText on every matching row at once.

Pair that with a per-iteration requery (limit 1) of the work list
so posts whose replies were already filled in as a side-effect of
a previous post's update are skipped without burning an API call.
This commit is contained in:
jim
2026-05-07 14:19:54 -05:00
parent 64a914c14d
commit ac79727040
3 changed files with 36 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/settings.local.json
+7
View File
@@ -0,0 +1,7 @@
{
"permissions": {
"allow": [
"Bash(dotnet build *)"
]
}
}
URLNotesGrabberCORE/DataAccess.cs
+2 -2
View File
@@ -1478,8 +1478,8 @@ namespace URLNotesGrabberCORE
connection.Open();
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
// Tolerance window on TimeStamp absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
// Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
URLNotesGrabberCORE/Program.cs
+27 -25
View File
@@ -562,14 +562,16 @@ namespace URLNotesGrabberCORE
Console.WriteLine();
int totalProcessedCount = 0;
int totalReplies = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
Console.WriteLine($"[Reply Text] Total replies to process: {totalReplies}");
Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
Console.WriteLine();
while (totalProcessedCount < totalReplies)
while (true)
{
var batch = DataAccess.GetRepliesWithFilledText();
// Re-query each iteration so posts whose replies got filled in as a side-effect
// of a previous post's update are skipped without burning an API call.
var batch = DataAccess.GetRepliesWithFilledText(limit: 1);
if (batch is null || batch.Count == 0)
{
@@ -577,31 +579,31 @@ namespace URLNotesGrabberCORE
break;
}
foreach (var reply in batch)
var reply = batch[0];
var blogName = reply.Item1;
var postID = reply.Item2;
var timestamp = reply.Item3;
await FetchAndStoreReplyText(blogName, postID, timestamp);
totalProcessedCount++;
int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
double completionPct = initialTotal > 0
? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
: 100.0;
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
Console.ForegroundColor = previousColor;
if (totalProcessedCount % 50 == 0)
{
var blogName = (reply as Tuple<string, long, long>).Item1;
var postID = (reply as Tuple<string, long, long>).Item2;
var timestamp = (reply as Tuple<string, long, long>).Item3;
await FetchAndStoreReplyText(blogName, postID, timestamp);
totalProcessedCount++;
int remaining = totalReplies - totalProcessedCount;
double completionPct = (totalProcessedCount / (double)totalReplies) * 100.0;
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($"[{remaining} remaining] [{totalProcessedCount}/{totalReplies}] ({completionPct:F1}%)");
Console.ForegroundColor = previousColor;
if (totalProcessedCount % 50 == 0 && totalProcessedCount < totalReplies)
{
await Task.Delay(2000);
}
await Task.Delay(2000);
}
}
Console.WriteLine($"[Reply Text] Complete. Total processed: {totalProcessedCount} replies.");
Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
}
catch (Exception ex)
{