feat: fan reply updates across reblog chains, requery per post
A reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it. Drop PostID from the UpdateNoteReplyText WHERE clause so a single API hit fills in the replyText on every matching row at once. Pair that with a per-iteration requery (limit 1) of the work list so posts whose replies were already filled in as a side-effect of a previous post's update are skipped without burning an API call.
This commit is contained in:
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"permissions": {
|
||||||
|
"allow": [
|
||||||
|
"Bash(dotnet build *)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1478,8 +1478,8 @@ namespace URLNotesGrabberCORE
|
|||||||
connection.Open();
|
connection.Open();
|
||||||
|
|
||||||
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
|
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
|
||||||
// Tolerance window on TimeStamp absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
|
// Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
|
||||||
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
|
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
|
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
|
||||||
|
|||||||
@@ -562,14 +562,16 @@ namespace URLNotesGrabberCORE
|
|||||||
Console.WriteLine();
|
Console.WriteLine();
|
||||||
|
|
||||||
int totalProcessedCount = 0;
|
int totalProcessedCount = 0;
|
||||||
int totalReplies = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
||||||
|
|
||||||
Console.WriteLine($"[Reply Text] Total replies to process: {totalReplies}");
|
Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
|
||||||
Console.WriteLine();
|
Console.WriteLine();
|
||||||
|
|
||||||
while (totalProcessedCount < totalReplies)
|
while (true)
|
||||||
{
|
{
|
||||||
var batch = DataAccess.GetRepliesWithFilledText();
|
// Re-query each iteration so posts whose replies got filled in as a side-effect
|
||||||
|
// of a previous post's update are skipped without burning an API call.
|
||||||
|
var batch = DataAccess.GetRepliesWithFilledText(limit: 1);
|
||||||
|
|
||||||
if (batch is null || batch.Count == 0)
|
if (batch is null || batch.Count == 0)
|
||||||
{
|
{
|
||||||
@@ -577,31 +579,31 @@ namespace URLNotesGrabberCORE
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var reply in batch)
|
var reply = batch[0];
|
||||||
{
|
var blogName = reply.Item1;
|
||||||
var blogName = (reply as Tuple<string, long, long>).Item1;
|
var postID = reply.Item2;
|
||||||
var postID = (reply as Tuple<string, long, long>).Item2;
|
var timestamp = reply.Item3;
|
||||||
var timestamp = (reply as Tuple<string, long, long>).Item3;
|
|
||||||
|
|
||||||
await FetchAndStoreReplyText(blogName, postID, timestamp);
|
await FetchAndStoreReplyText(blogName, postID, timestamp);
|
||||||
totalProcessedCount++;
|
totalProcessedCount++;
|
||||||
|
|
||||||
int remaining = totalReplies - totalProcessedCount;
|
int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
||||||
double completionPct = (totalProcessedCount / (double)totalReplies) * 100.0;
|
double completionPct = initialTotal > 0
|
||||||
|
? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
|
||||||
|
: 100.0;
|
||||||
|
|
||||||
var previousColor = Console.ForegroundColor;
|
var previousColor = Console.ForegroundColor;
|
||||||
Console.ForegroundColor = ConsoleColor.Cyan;
|
Console.ForegroundColor = ConsoleColor.Cyan;
|
||||||
Console.WriteLine($"[{remaining} remaining] [{totalProcessedCount}/{totalReplies}] ({completionPct:F1}%)");
|
Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
|
||||||
Console.ForegroundColor = previousColor;
|
Console.ForegroundColor = previousColor;
|
||||||
|
|
||||||
if (totalProcessedCount % 50 == 0 && totalProcessedCount < totalReplies)
|
if (totalProcessedCount % 50 == 0)
|
||||||
{
|
{
|
||||||
await Task.Delay(2000);
|
await Task.Delay(2000);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Console.WriteLine($"[Reply Text] Complete. Total processed: {totalProcessedCount} replies.");
|
Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user