diff --git a/.gitignore b/.gitignore index 9491a2f..18ec4d3 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,5 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd \ No newline at end of file +FodyWeavers.xsd +/URLNotesGrabberCORE/TL.db diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index dbaf486..f63c904 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -2188,9 +2188,15 @@ namespace URLNotesGrabberCORE } } - public static async Task GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp) + public static async Task GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp) { - var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}"; + var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=conversation"; + URL = URL.Replace("[0]", blog).Replace("[1]", postID.ToString()); + if (timestamp > 0) + { + URL += $"&before_timestamp={timestamp}"; + await Task.Delay(100); + } using (var client = BuildClient(key, URL)) { @@ -2199,7 +2205,7 @@ namespace URLNotesGrabberCORE var myJsonResponse = response.Content ?? string.Empty; Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}"); - var myDeserializedClass = new PostsRoot(); + var myDeserializedClass = new Root(); // Check if response is successful and contains JSON if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse)) @@ -2225,17 +2231,138 @@ namespace URLNotesGrabberCORE try { - var deserializedResult = JsonConvert.DeserializeObject(myJsonResponse); + var deserializedResult = JsonConvert.DeserializeObject(myJsonResponse); if (deserializedResult != null) { myDeserializedClass = deserializedResult; myDeserializedClass.rawJson = myJsonResponse; + + bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429; + bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0; + + if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))) + { + if (response?.Headers != null) + { + bool checkResetLocal = false; + foreach (var header in response.Headers) + { + string? headerName = header?.Name; + string? headerValue = header?.Value?.ToString(); + if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue)) + continue; + + if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int retrySecs)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs); + else if (DateTimeOffset.TryParse(headerValue, out var dto)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds)); + } + + if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch)) + { + var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs); + } + + if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0") + checkResetLocal = true; + + if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0) + { + if (int.TryParse(headerValue, out int resetValue)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue); + else if (long.TryParse(headerValue, out long epochVal)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds())); + } + } + } + + myDeserializedClass.statusCode = "TooManyRequests"; + } } } catch (Exception ex) { Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}"); Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}"); + + if (!response.IsSuccessful) + { + string? statusStr = null; + try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; } + Console.WriteLine($"{statusStr}\t{response?.StatusDescription}"); + if (!string.IsNullOrEmpty(statusStr)) + myDeserializedClass.statusCode = statusStr; + + bool checkReset = false; + + if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null) + { + bool foundRateLimitHeader = false; + foreach (var header in response.Headers) + { + if (header.Name != null && header.Value != null) + { + Console.WriteLine($"{header.Name} - {header.Value}"); + var headerValue = header.Value?.ToString(); + if (!string.IsNullOrEmpty(headerValue)) + { + if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int retrySecs)) + { + if (myDeserializedClass.retryInSeconds < retrySecs) + myDeserializedClass.retryInSeconds = retrySecs; + } + else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto)) + { + var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + } + + if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int resetValue)) + { + if (myDeserializedClass.retryInSeconds < resetValue) + myDeserializedClass.retryInSeconds = resetValue; + } + else if (long.TryParse(headerValue, out long epochVal)) + { + var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + } + + if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0) + { + if (long.TryParse(headerValue, out long epoch)) + { + var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + foundRateLimitHeader = true; + } + } + if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0") + checkReset = true; + else + checkReset = false; + } + + if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)) + { + myDeserializedClass.statusCode = "TooManyRequests"; + } + } + } + } } return myDeserializedClass; diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 67feb29..a4f673c 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -464,41 +464,27 @@ namespace URLNotesGrabberCORE return; } - if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0) + if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0) { - Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}"); + Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}"); Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}"); Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}"); if (postsResponse?.response != null) { - Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}"); - } - if (!string.IsNullOrEmpty(postsResponse?.rawJson)) - { - Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}"); + Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}"); } // Mark all replies for this post with '?' to indicate API returned no posts - Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response"); + Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response"); DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?"); return; } - // Get the first (and should be only) post - var post = postsResponse.response.posts.FirstOrDefault(); - if (post?.notes == null) - { - Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}"); - return; - } - - Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}"); - // Update each reply with its text int replyCount = 0; int skippedCount = 0; - foreach (var note in post.notes) + foreach (var note in postsResponse.response.notes) { if (note.type == "reply") { @@ -519,6 +505,18 @@ namespace URLNotesGrabberCORE Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text"); } } + else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text)) + { + // Handle reblogs with comment + DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); + replyCount++; + + // Output the reblog comment being stored + string displayText = note.reply_text.Length > 100 + ? note.reply_text.Substring(0, 100) + "..." + : note.reply_text; + Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)"); + } } if (replyCount > 0) diff --git a/URLNotesGrabberCORE/ResponseNotes.cs b/URLNotesGrabberCORE/ResponseNotes.cs index 066adff..c80f66c 100644 --- a/URLNotesGrabberCORE/ResponseNotes.cs +++ b/URLNotesGrabberCORE/ResponseNotes.cs @@ -5,12 +5,12 @@ using System.Text; using System.Threading.Tasks; using Newtonsoft.Json; - namespace URLNotesGrabberCORE { internal class ResponseNotes { } + // Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); public class AvatarUrl { @@ -42,7 +42,7 @@ namespace URLNotesGrabberCORE public class Note { public string type { get; set; } - public int timestamp { get; set; } + public long timestamp { get; set; } public string blog_name { get; set; } public string blog_uuid { get; set; } public string blog_url { get; set; } @@ -51,6 +51,7 @@ namespace URLNotesGrabberCORE public AvatarUrl avatar_url { get; set; } public string post_id { get; set; } public string reblog_parent_blog_name { get; set; } + public string reply_text { get; set; } } public class QueryParams