From 1da8a702bae4b0130b73e4266eae1633e9a8abc4 Mon Sep 17 00:00:00 2001 From: jim Date: Thu, 7 May 2026 13:54:54 -0500 Subject: [PATCH] fix: pull reply_text from Tumblr in -replies mode Wrap FetchAndStoreReplyText in a per-post pagination loop (up to 10 pages, advancing before_timestamp via the last note's timestamp) so posts with >50 notes are fully walked. Log raw response (meta + first 500 chars of JSON) when a page returns no notes so empty results are diagnosable. Stop blanket-marking every reply on a post with '?' on the first empty response - rows stay '.' and remain retry-eligible; only individual replies that come back with empty reply_text are marked '?'. --- URLNotesGrabberCORE/Program.cs | 151 ++++++++++++++++----------------- 1 file changed, 73 insertions(+), 78 deletions(-) diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 1ba7baf..93a09bb 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -449,103 +449,98 @@ namespace URLNotesGrabberCORE static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp) { + const int MaxPages = 10; try { Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}"); - await Task.Delay(2000); - var key = ApiKeyPool.GetCurrentKey(); - var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp); - if (postsResponse?.statusCode == "TooManyRequests") - { - int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60; - ApiKeyPool.MarkRateLimited(key, retry); - Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key"); - return; - } - - if (postsResponse?.meta?.status != 429) - ApiKeyPool.MarkAvailable(key); - - if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0) - { - Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}"); - Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}"); - Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}"); - if (postsResponse?.response != null) - { - Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}"); - } - - var previousColor = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response"); - Console.ForegroundColor = previousColor; - DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?"); - - return; - } - - // Update each reply with its text int replyCount = 0; - int skippedCount = 0; - foreach (var note in postsResponse.response.notes) + int emptyReplyCount = 0; + long pageTimestamp = timestamp; + int page = 0; + + while (page < MaxPages) { - if (note.type == "reply") + page++; + await Task.Delay(2000); + var key = ApiKeyPool.GetCurrentKey(); + var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp); + + if (postsResponse?.statusCode == "TooManyRequests") { - if (!string.IsNullOrEmpty(note.reply_text)) + int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60; + ApiKeyPool.MarkRateLimited(key, retry); + Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key"); + return; + } + + if (postsResponse?.meta?.status != 429) + ApiKeyPool.MarkAvailable(key); + + if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0) + { + var prevColor = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine($"[Reply Text] No notes returned for {blogName}/{postID} on page {page} (before_timestamp={pageTimestamp})"); + Console.ForegroundColor = prevColor; + Console.WriteLine($"[Reply Text] meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}"); + var raw = postsResponse?.rawJson ?? string.Empty; + if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]"; + Console.WriteLine($"[Reply Text] raw: {raw}"); + break; + } + + long lastNoteTimestamp = 0; + foreach (var note in postsResponse.response.notes) + { + if (note.timestamp > 0) + lastNoteTimestamp = note.timestamp; + + if (note.type == "reply") + { + if (!string.IsNullOrEmpty(note.reply_text)) + { + DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); + replyCount++; + + string displayText = note.reply_text.Length > 100 + ? note.reply_text.Substring(0, 100) + "..." + : note.reply_text; + var previousColor = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine($" [{note.blog_name}] {displayText}"); + Console.ForegroundColor = previousColor; + } + else + { + DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?"); + emptyReplyCount++; + Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'"); + } + } + else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text)) { DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); replyCount++; - - // Output the reply text being stored - string displayText = note.reply_text.Length > 100 - ? note.reply_text.Substring(0, 100) + "..." + + string displayText = note.reply_text.Length > 100 + ? note.reply_text.Substring(0, 100) + "..." : note.reply_text; var previousColor = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Green; - Console.WriteLine($" [{note.blog_name}] {displayText}"); + Console.ForegroundColor = ConsoleColor.Cyan; + Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)"); Console.ForegroundColor = previousColor; } - else - { - skippedCount++; - Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text"); - } } - else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text)) + + if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp)) { - // Handle reblogs with comment - DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); - replyCount++; - - // Output the reblog comment being stored - string displayText = note.reply_text.Length > 100 - ? note.reply_text.Substring(0, 100) + "..." - : note.reply_text; - var previousColor = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)"); - Console.ForegroundColor = previousColor; + break; } + pageTimestamp = lastNoteTimestamp; } - if (replyCount > 0) - { - Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}"); - } - else - { - Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})"); - } - - // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text - Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}"); - //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?"); - //if (cleanupCount > 0) - //{ - // Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}"); - //} + Console.WriteLine($"[Reply Text] Done {blogName}/{postID}: updated={replyCount}, empty='?'={emptyReplyCount}, pages={page}"); } catch (Exception ex) {