✅ Uses mode=conversation to fetch only notes with text
✅ Captures replies and reblogs with comment ✅ Ignores rollup_notes field (as requested) ✅ Maintains rate limiting and error handling ✅ Console output shows both reply and reblog comment
This commit is contained in:
@@ -361,3 +361,4 @@ MigrationBackup/
|
||||
|
||||
# Fody - auto-generated XML schema
|
||||
FodyWeavers.xsd
|
||||
/URLNotesGrabberCORE/TL.db
|
||||
|
||||
@@ -2188,9 +2188,15 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<PostsRoot> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
|
||||
public static async Task<Root> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
|
||||
{
|
||||
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}";
|
||||
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=conversation";
|
||||
URL = URL.Replace("[0]", blog).Replace("[1]", postID.ToString());
|
||||
if (timestamp > 0)
|
||||
{
|
||||
URL += $"&before_timestamp={timestamp}";
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
using (var client = BuildClient(key, URL))
|
||||
{
|
||||
@@ -2199,7 +2205,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
var myJsonResponse = response.Content ?? string.Empty;
|
||||
Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
var myDeserializedClass = new PostsRoot();
|
||||
var myDeserializedClass = new Root();
|
||||
|
||||
// Check if response is successful and contains JSON
|
||||
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
|
||||
@@ -2225,17 +2231,138 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
try
|
||||
{
|
||||
var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
|
||||
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
||||
if (deserializedResult != null)
|
||||
{
|
||||
myDeserializedClass = deserializedResult;
|
||||
myDeserializedClass.rawJson = myJsonResponse;
|
||||
|
||||
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
||||
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
||||
{
|
||||
if (response?.Headers != null)
|
||||
{
|
||||
bool checkResetLocal = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
string? headerName = header?.Name;
|
||||
string? headerValue = header?.Value?.ToString();
|
||||
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
||||
continue;
|
||||
|
||||
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
||||
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
||||
}
|
||||
|
||||
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
||||
}
|
||||
|
||||
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
||||
checkResetLocal = true;
|
||||
|
||||
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
|
||||
Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
|
||||
|
||||
if (!response.IsSuccessful)
|
||||
{
|
||||
string? statusStr = null;
|
||||
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
||||
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
||||
if (!string.IsNullOrEmpty(statusStr))
|
||||
myDeserializedClass.statusCode = statusStr;
|
||||
|
||||
bool checkReset = false;
|
||||
|
||||
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
||||
{
|
||||
bool foundRateLimitHeader = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
if (header.Name != null && header.Value != null)
|
||||
{
|
||||
Console.WriteLine($"{header.Name} - {header.Value}");
|
||||
var headerValue = header.Value?.ToString();
|
||||
if (!string.IsNullOrEmpty(headerValue))
|
||||
{
|
||||
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < retrySecs)
|
||||
myDeserializedClass.retryInSeconds = retrySecs;
|
||||
}
|
||||
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
||||
{
|
||||
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < resetValue)
|
||||
myDeserializedClass.retryInSeconds = resetValue;
|
||||
}
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
foundRateLimitHeader = true;
|
||||
}
|
||||
}
|
||||
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
||||
checkReset = true;
|
||||
else
|
||||
checkReset = false;
|
||||
}
|
||||
|
||||
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
||||
{
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return myDeserializedClass;
|
||||
|
||||
@@ -464,41 +464,27 @@ namespace URLNotesGrabberCORE
|
||||
return;
|
||||
}
|
||||
|
||||
if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
|
||||
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
|
||||
Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}");
|
||||
Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
|
||||
Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
|
||||
if (postsResponse?.response != null)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
|
||||
}
|
||||
if (!string.IsNullOrEmpty(postsResponse?.rawJson))
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
|
||||
Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
|
||||
}
|
||||
|
||||
// Mark all replies for this post with '?' to indicate API returned no posts
|
||||
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
|
||||
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response");
|
||||
DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the first (and should be only) post
|
||||
var post = postsResponse.response.posts.FirstOrDefault();
|
||||
if (post?.notes == null)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");
|
||||
|
||||
// Update each reply with its text
|
||||
int replyCount = 0;
|
||||
int skippedCount = 0;
|
||||
foreach (var note in post.notes)
|
||||
foreach (var note in postsResponse.response.notes)
|
||||
{
|
||||
if (note.type == "reply")
|
||||
{
|
||||
@@ -519,6 +505,18 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
|
||||
}
|
||||
}
|
||||
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
|
||||
{
|
||||
// Handle reblogs with comment
|
||||
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
|
||||
replyCount++;
|
||||
|
||||
// Output the reblog comment being stored
|
||||
string displayText = note.reply_text.Length > 100
|
||||
? note.reply_text.Substring(0, 100) + "..."
|
||||
: note.reply_text;
|
||||
Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)");
|
||||
}
|
||||
}
|
||||
|
||||
if (replyCount > 0)
|
||||
|
||||
@@ -5,12 +5,12 @@ using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
|
||||
namespace URLNotesGrabberCORE
|
||||
{
|
||||
internal class ResponseNotes
|
||||
{
|
||||
}
|
||||
|
||||
// Root myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
||||
public class AvatarUrl
|
||||
{
|
||||
@@ -42,7 +42,7 @@ namespace URLNotesGrabberCORE
|
||||
public class Note
|
||||
{
|
||||
public string type { get; set; }
|
||||
public int timestamp { get; set; }
|
||||
public long timestamp { get; set; }
|
||||
public string blog_name { get; set; }
|
||||
public string blog_uuid { get; set; }
|
||||
public string blog_url { get; set; }
|
||||
@@ -51,6 +51,7 @@ namespace URLNotesGrabberCORE
|
||||
public AvatarUrl avatar_url { get; set; }
|
||||
public string post_id { get; set; }
|
||||
public string reblog_parent_blog_name { get; set; }
|
||||
public string reply_text { get; set; }
|
||||
}
|
||||
|
||||
public class QueryParams
|
||||
|
||||
Reference in New Issue
Block a user