jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✅ Uses mode=conversation to fetch only notes with text

Browse Source 
✅ Captures replies and reblogs with comment
✅ Ignores rollup_notes field (as requested)
✅ Maintains rate limiting and error handling
✅ Console output shows both reply and reblog comment
This commit is contained in:
jim
2026-05-05 11:49:10 -05:00
parent 354bb9cc8e
commit abdfe30203
4 changed files with 153 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -360,4 +360,5 @@ MigrationBackup/
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd
FodyWeavers.xsd
/URLNotesGrabberCORE/TL.db
URLNotesGrabberCORE/DataAccess.cs
+131 -4
View File
@@ -2188,9 +2188,15 @@ namespace URLNotesGrabberCORE
}
}
public static async Task<PostsRoot> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
public static async Task<Root> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
{
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}&notes_info=true&before_timestamp={timestamp}";
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=conversation";
URL = URL.Replace("[0]", blog).Replace("[1]", postID.ToString());
if (timestamp > 0)
{
URL += $"&before_timestamp={timestamp}";
await Task.Delay(100);
}
using (var client = BuildClient(key, URL))
{
@@ -2199,7 +2205,7 @@ namespace URLNotesGrabberCORE
var myJsonResponse = response.Content ?? string.Empty;
Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
var myDeserializedClass = new PostsRoot();
var myDeserializedClass = new Root();
// Check if response is successful and contains JSON
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
@@ -2225,17 +2231,138 @@ namespace URLNotesGrabberCORE
try
{
var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
if (deserializedResult != null)
{
myDeserializedClass = deserializedResult;
myDeserializedClass.rawJson = myJsonResponse;
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
{
if (response?.Headers != null)
{
bool checkResetLocal = false;
foreach (var header in response.Headers)
{
string? headerName = header?.Name;
string? headerValue = header?.Value?.ToString();
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
continue;
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int retrySecs))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
else if (DateTimeOffset.TryParse(headerValue, out var dto))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
}
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
{
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
}
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
checkResetLocal = true;
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (int.TryParse(headerValue, out int resetValue))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
else if (long.TryParse(headerValue, out long epochVal))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
}
}
}
myDeserializedClass.statusCode = "TooManyRequests";
}
}
}
catch (Exception ex)
{
Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
if (!response.IsSuccessful)
{
string? statusStr = null;
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
if (!string.IsNullOrEmpty(statusStr))
myDeserializedClass.statusCode = statusStr;
bool checkReset = false;
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
{
bool foundRateLimitHeader = false;
foreach (var header in response.Headers)
{
if (header.Name != null && header.Value != null)
{
Console.WriteLine($"{header.Name} - {header.Value}");
var headerValue = header.Value?.ToString();
if (!string.IsNullOrEmpty(headerValue))
{
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int retrySecs))
{
if (myDeserializedClass.retryInSeconds < retrySecs)
myDeserializedClass.retryInSeconds = retrySecs;
}
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
{
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
}
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int resetValue))
{
if (myDeserializedClass.retryInSeconds < resetValue)
myDeserializedClass.retryInSeconds = resetValue;
}
else if (long.TryParse(headerValue, out long epochVal))
{
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
}
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (long.TryParse(headerValue, out long epoch))
{
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
foundRateLimitHeader = true;
}
}
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
checkReset = true;
else
checkReset = false;
}
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
{
myDeserializedClass.statusCode = "TooManyRequests";
}
}
}
}
}
return myDeserializedClass;
URLNotesGrabberCORE/Program.cs
+17 -19
View File
@@ -464,41 +464,27 @@ namespace URLNotesGrabberCORE
return;
}
if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
{
Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}");
Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
if (postsResponse?.response != null)
{
Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
}
if (!string.IsNullOrEmpty(postsResponse?.rawJson))
{
Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
}
// Mark all replies for this post with '?' to indicate API returned no posts
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response");
DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
return;
}
// Get the first (and should be only) post
var post = postsResponse.response.posts.FirstOrDefault();
if (post?.notes == null)
{
Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
return;
}
Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");
// Update each reply with its text
int replyCount = 0;
int skippedCount = 0;
foreach (var note in post.notes)
foreach (var note in postsResponse.response.notes)
{
if (note.type == "reply")
{
@@ -519,6 +505,18 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
}
}
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
{
// Handle reblogs with comment
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
replyCount++;
// Output the reblog comment being stored
string displayText = note.reply_text.Length > 100
? note.reply_text.Substring(0, 100) + "..."
: note.reply_text;
Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)");
}
}
if (replyCount > 0)
URLNotesGrabberCORE/ResponseNotes.cs
+3 -2
View File
@@ -5,12 +5,12 @@ using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace URLNotesGrabberCORE
{
internal class ResponseNotes
{
}
// Root myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
public class AvatarUrl
{
@@ -42,7 +42,7 @@ namespace URLNotesGrabberCORE
public class Note
{
public string type { get; set; }
public int timestamp { get; set; }
public long timestamp { get; set; }
public string blog_name { get; set; }
public string blog_uuid { get; set; }
public string blog_url { get; set; }
@@ -51,6 +51,7 @@ namespace URLNotesGrabberCORE
public AvatarUrl avatar_url { get; set; }
public string post_id { get; set; }
public string reblog_parent_blog_name { get; set; }
public string reply_text { get; set; }
}
public class QueryParams