jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: pull reply_text from Tumblr in -replies mode

Browse Source 
Wrap FetchAndStoreReplyText in a per-post pagination loop (up to 10
pages, advancing before_timestamp via the last note's timestamp) so
posts with >50 notes are fully walked. Log raw response (meta + first
500 chars of JSON) when a page returns no notes so empty results are
diagnosable. Stop blanket-marking every reply on a post with '?' on
the first empty response - rows stay '.' and remain retry-eligible;
only individual replies that come back with empty reply_text are
marked '?'.
This commit is contained in:
jim
2026-05-07 13:54:54 -05:00
parent b70f356d24
commit 1da8a702ba
1 changed files with 73 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+66 -71
View File
@@ -449,103 +449,98 @@ namespace URLNotesGrabberCORE
static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
{
const int MaxPages = 10;
try
{
Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
await Task.Delay(2000);
var key = ApiKeyPool.GetCurrentKey();
var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp);
if (postsResponse?.statusCode == "TooManyRequests")
int replyCount = 0;
int emptyReplyCount = 0;
long pageTimestamp = timestamp;
int page = 0;
while (page < MaxPages)
{
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
return;
}
page++;
await Task.Delay(2000);
var key = ApiKeyPool.GetCurrentKey();
var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);
if (postsResponse?.meta?.status != 429)
ApiKeyPool.MarkAvailable(key);
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
{
Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}");
Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
if (postsResponse?.response != null)
if (postsResponse?.statusCode == "TooManyRequests")
{
Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
return;
}
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response");
Console.ForegroundColor = previousColor;
DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
if (postsResponse?.meta?.status != 429)
ApiKeyPool.MarkAvailable(key);
return;
}
// Update each reply with its text
int replyCount = 0;
int skippedCount = 0;
foreach (var note in postsResponse.response.notes)
{
if (note.type == "reply")
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
{
if (!string.IsNullOrEmpty(note.reply_text))
var prevColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"[Reply Text] No notes returned for {blogName}/{postID} on page {page} (before_timestamp={pageTimestamp})");
Console.ForegroundColor = prevColor;
Console.WriteLine($"[Reply Text] meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
var raw = postsResponse?.rawJson ?? string.Empty;
if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
Console.WriteLine($"[Reply Text] raw: {raw}");
break;
}
long lastNoteTimestamp = 0;
foreach (var note in postsResponse.response.notes)
{
if (note.timestamp > 0)
lastNoteTimestamp = note.timestamp;
if (note.type == "reply")
{
if (!string.IsNullOrEmpty(note.reply_text))
{
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
replyCount++;
string displayText = note.reply_text.Length > 100
? note.reply_text.Substring(0, 100) + "..."
: note.reply_text;
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" [{note.blog_name}] {displayText}");
Console.ForegroundColor = previousColor;
}
else
{
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
emptyReplyCount++;
Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
}
}
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
{
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
replyCount++;
// Output the reply text being stored
string displayText = note.reply_text.Length > 100
? note.reply_text.Substring(0, 100) + "..."
: note.reply_text;
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" [{note.blog_name}] {displayText}");
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)");
Console.ForegroundColor = previousColor;
}
else
{
skippedCount++;
Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
}
}
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
{
// Handle reblogs with comment
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
replyCount++;
// Output the reblog comment being stored
string displayText = note.reply_text.Length > 100
? note.reply_text.Substring(0, 100) + "..."
: note.reply_text;
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)");
Console.ForegroundColor = previousColor;
break;
}
pageTimestamp = lastNoteTimestamp;
}
if (replyCount > 0)
{
Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
}
else
{
Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
}
// Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
//int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
//if (cleanupCount > 0)
//{
// Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
//}
Console.WriteLine($"[Reply Text] Done {blogName}/{postID}: updated={replyCount}, empty='?'={emptyReplyCount}, pages={page}");
}
catch (Exception ex)
{