jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: pull reply_text from Tumblr in -replies mode

Browse Source 
Wrap FetchAndStoreReplyText in a per-post pagination loop (up to 10
pages, advancing before_timestamp via the last note's timestamp) so
posts with >50 notes are fully walked. Log raw response (meta + first
500 chars of JSON) when a page returns no notes so empty results are
diagnosable. Stop blanket-marking every reply on a post with '?' on
the first empty response - rows stay '.' and remain retry-eligible;
only individual replies that come back with empty reply_text are
marked '?'.
This commit is contained in:
jim
2026-05-07 13:54:54 -05:00
parent b70f356d24
commit 1da8a702ba
1 changed files with 73 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+30 -35
View File
@@ -449,12 +449,22 @@ namespace URLNotesGrabberCORE
static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp) static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
{ {
const int MaxPages = 10;
try try
{ {
Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}"); Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
int replyCount = 0;
int emptyReplyCount = 0;
long pageTimestamp = timestamp;
int page = 0;
while (page < MaxPages)
{
page++;
await Task.Delay(2000); await Task.Delay(2000);
var key = ApiKeyPool.GetCurrentKey(); var key = ApiKeyPool.GetCurrentKey();
var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp); var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);
if (postsResponse?.statusCode == "TooManyRequests") if (postsResponse?.statusCode == "TooManyRequests")
{ {
@@ -469,28 +479,23 @@ namespace URLNotesGrabberCORE
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0) if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
{ {
Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}"); var prevColor = Console.ForegroundColor;
Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
if (postsResponse?.response != null)
{
Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
}
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow; Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response"); Console.WriteLine($"[Reply Text] No notes returned for {blogName}/{postID} on page {page} (before_timestamp={pageTimestamp})");
Console.ForegroundColor = previousColor; Console.ForegroundColor = prevColor;
DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?"); Console.WriteLine($"[Reply Text] meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
var raw = postsResponse?.rawJson ?? string.Empty;
return; if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
Console.WriteLine($"[Reply Text] raw: {raw}");
break;
} }
// Update each reply with its text long lastNoteTimestamp = 0;
int replyCount = 0;
int skippedCount = 0;
foreach (var note in postsResponse.response.notes) foreach (var note in postsResponse.response.notes)
{ {
if (note.timestamp > 0)
lastNoteTimestamp = note.timestamp;
if (note.type == "reply") if (note.type == "reply")
{ {
if (!string.IsNullOrEmpty(note.reply_text)) if (!string.IsNullOrEmpty(note.reply_text))
@@ -498,7 +503,6 @@ namespace URLNotesGrabberCORE
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
replyCount++; replyCount++;
// Output the reply text being stored
string displayText = note.reply_text.Length > 100 string displayText = note.reply_text.Length > 100
? note.reply_text.Substring(0, 100) + "..." ? note.reply_text.Substring(0, 100) + "..."
: note.reply_text; : note.reply_text;
@@ -509,17 +513,16 @@ namespace URLNotesGrabberCORE
} }
else else
{ {
skippedCount++; DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text"); emptyReplyCount++;
Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
} }
} }
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text)) else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
{ {
// Handle reblogs with comment
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
replyCount++; replyCount++;
// Output the reblog comment being stored
string displayText = note.reply_text.Length > 100 string displayText = note.reply_text.Length > 100
? note.reply_text.Substring(0, 100) + "..." ? note.reply_text.Substring(0, 100) + "..."
: note.reply_text; : note.reply_text;
@@ -530,22 +533,14 @@ namespace URLNotesGrabberCORE
} }
} }
if (replyCount > 0) if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
{ {
Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}"); break;
} }
else pageTimestamp = lastNoteTimestamp;
{
Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
} }
// Mark any remaining replies with '.' as '?' to indicate they were processed but had no text Console.WriteLine($"[Reply Text] Done {blogName}/{postID}: updated={replyCount}, empty='?'={emptyReplyCount}, pages={page}");
Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
//int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
//if (cleanupCount > 0)
//{
// Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
//}
} }
catch (Exception ex) catch (Exception ex)
{ {