jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✅ Uses mode=conversation to fetch only notes with text

Browse Source 
✅ Captures replies and reblogs with comment
✅ Ignores rollup_notes field (as requested)
✅ Maintains rate limiting and error handling
✅ Console output shows both reply and reblog comment
This commit is contained in:
jim
2026-05-05 11:49:10 -05:00
parent 354bb9cc8e
commit abdfe30203
4 changed files with 153 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+131 -4
View File
@@ -2188,9 +2188,15 @@ namespace URLNotesGrabberCORE
}
}
public static async Task<PostsRoot> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
public static async Task<Root> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
{
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}&notes_info=true&before_timestamp={timestamp}";
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=conversation";
URL = URL.Replace("[0]", blog).Replace("[1]", postID.ToString());
if (timestamp > 0)
{
URL += $"&before_timestamp={timestamp}";
await Task.Delay(100);
}
using (var client = BuildClient(key, URL))
{
@@ -2199,7 +2205,7 @@ namespace URLNotesGrabberCORE
var myJsonResponse = response.Content ?? string.Empty;
Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
var myDeserializedClass = new PostsRoot();
var myDeserializedClass = new Root();
// Check if response is successful and contains JSON
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
@@ -2225,17 +2231,138 @@ namespace URLNotesGrabberCORE
try
{
var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
if (deserializedResult != null)
{
myDeserializedClass = deserializedResult;
myDeserializedClass.rawJson = myJsonResponse;
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
{
if (response?.Headers != null)
{
bool checkResetLocal = false;
foreach (var header in response.Headers)
{
string? headerName = header?.Name;
string? headerValue = header?.Value?.ToString();
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
continue;
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int retrySecs))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
else if (DateTimeOffset.TryParse(headerValue, out var dto))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
}
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
{
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
}
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
checkResetLocal = true;
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (int.TryParse(headerValue, out int resetValue))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
else if (long.TryParse(headerValue, out long epochVal))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
}
}
}
myDeserializedClass.statusCode = "TooManyRequests";
}
}
}
catch (Exception ex)
{
Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
if (!response.IsSuccessful)
{
string? statusStr = null;
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
if (!string.IsNullOrEmpty(statusStr))
myDeserializedClass.statusCode = statusStr;
bool checkReset = false;
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
{
bool foundRateLimitHeader = false;
foreach (var header in response.Headers)
{
if (header.Name != null && header.Value != null)
{
Console.WriteLine($"{header.Name} - {header.Value}");
var headerValue = header.Value?.ToString();
if (!string.IsNullOrEmpty(headerValue))
{
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int retrySecs))
{
if (myDeserializedClass.retryInSeconds < retrySecs)
myDeserializedClass.retryInSeconds = retrySecs;
}
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
{
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
}
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int resetValue))
{
if (myDeserializedClass.retryInSeconds < resetValue)
myDeserializedClass.retryInSeconds = resetValue;
}
else if (long.TryParse(headerValue, out long epochVal))
{
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
}
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (long.TryParse(headerValue, out long epoch))
{
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
foundRateLimitHeader = true;
}
}
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
checkReset = true;
else
checkReset = false;
}
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
{
myDeserializedClass.statusCode = "TooManyRequests";
}
}
}
}
}
return myDeserializedClass;