fix: retry transient CDN failures instead of failing the post
A non-JSON response body (CDN 403/5xx HTML, empty body, transport error) never reached the Tumblr API, so it says nothing about the post being fetched. These were recorded as FAILURE, which consumed the post's single attempt for the pass and cleared the API key's rate-limit flag on the way through. Classify them as Root.transientFailure and retry in place (1s/4s/10s) before skipping. Skipped posts stay unmarked in the DB so a later launch retries them. Ten consecutive transient failures now aborts the pass rather than skipping post-by-post against an edge refusing all traffic. Also: - MarkAvailable() only on a response that reached the API, and it is now a no-op when the key was not flagged (was writing to the DB and logging on every single call) - Only a real 429 counts as a rate limit; stop inferring one from X-RateLimit-* headers, which Tumblr sends on every response - Limiters pace with AcquireAsync instead of AttemptAcquire, which did not wait and aborted the run once a window was saturated - Throttle --collect and --likes from 300/min to 60/min - Log one line per transient failure instead of the HTML body and stack trace; keep full detail only for a 2xx that fails to parse - --collect returns exit 3 when a pass ends incomplete Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Claude Opus 4.8
@@ -2710,6 +2710,10 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
public static void MarkAvailable(ApiKeyConfig key)
|
||||
{
|
||||
// Called after every successful call; skip the write and the log line when nothing was flagged.
|
||||
if (GetRetryUntil(key) == 0)
|
||||
return;
|
||||
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
||||
@@ -2803,6 +2807,15 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
private static string FormatKeyLabel(ApiKeyConfig key) => $"[Key#{key.KeyNumber}]";
|
||||
|
||||
private static string SummarizeBody(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
return "(empty)";
|
||||
|
||||
var flat = System.Text.RegularExpressions.Regex.Replace(body, @"<[^>]+>|\s+", " ").Trim();
|
||||
return flat.Length <= 80 ? flat : flat.Substring(0, 80) + "...";
|
||||
}
|
||||
|
||||
private static int GetRetryDelaySecondsFromHeaders(IEnumerable<HeaderParameter>? headers)
|
||||
{
|
||||
if (headers == null)
|
||||
@@ -2881,144 +2894,63 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine($"{FormatKeyLabel(key)} {timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
var myDeserializedClass = new Root();
|
||||
|
||||
// Never reached the API: there is no body to interpret, so the post's state is still unknown.
|
||||
if (response.ResponseStatus != ResponseStatus.Completed)
|
||||
{
|
||||
myDeserializedClass.statusCode = response.ResponseStatus.ToString();
|
||||
myDeserializedClass.transientFailure = true;
|
||||
Console.WriteLine($"[Transient] {FormatKeyLabel(key)} transport {response.ResponseStatus}: {response.ErrorException?.Message}");
|
||||
return myDeserializedClass;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
||||
if (deserializedResult != null)
|
||||
if (deserializedResult == null)
|
||||
{
|
||||
myDeserializedClass = deserializedResult;
|
||||
myDeserializedClass.rawJson = myJsonResponse;
|
||||
// Empty body behind an HTTP status: an edge/proxy response, not the API.
|
||||
myDeserializedClass.statusCode = response.StatusCode.ToString();
|
||||
myDeserializedClass.transientFailure = true;
|
||||
Console.WriteLine($"[Transient] {FormatKeyLabel(key)} HTTP {(int)response.StatusCode} {response.StatusDescription} — empty body");
|
||||
return myDeserializedClass;
|
||||
}
|
||||
|
||||
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
|
||||
{
|
||||
myDeserializedClass.statusCode = "NotFound";
|
||||
}
|
||||
myDeserializedClass = deserializedResult;
|
||||
myDeserializedClass.rawJson = myJsonResponse;
|
||||
|
||||
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
||||
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
|
||||
{
|
||||
myDeserializedClass.statusCode = "NotFound";
|
||||
}
|
||||
|
||||
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
||||
{
|
||||
if (response?.Headers != null)
|
||||
{
|
||||
bool checkResetLocal = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
string? headerName = header?.Name;
|
||||
string? headerValue = header?.Value?.ToString();
|
||||
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
||||
continue;
|
||||
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
||||
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
||||
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
||||
}
|
||||
|
||||
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
||||
}
|
||||
|
||||
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
||||
checkResetLocal = true;
|
||||
|
||||
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, GetRetryDelaySecondsFromHeaders(response.Headers));
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Failed JSON: {myJsonResponse}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
// A body that will not parse came from infrastructure (CDN/proxy/WAF), not the Tumblr
|
||||
// API, so it says nothing about this post. Retryable, not a failure of the post itself.
|
||||
myDeserializedClass.statusCode = response.StatusCode.ToString();
|
||||
|
||||
if (!response.IsSuccessful)
|
||||
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
string? statusStr = null;
|
||||
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
||||
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
||||
if (!string.IsNullOrEmpty(statusStr))
|
||||
myDeserializedClass.statusCode = statusStr;
|
||||
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
else
|
||||
{
|
||||
myDeserializedClass.transientFailure = true;
|
||||
Console.WriteLine($"[Transient] {FormatKeyLabel(key)} HTTP {(int)response.StatusCode} {response.StatusDescription} — unparseable body: {SummarizeBody(myJsonResponse)}");
|
||||
|
||||
bool checkReset = false;
|
||||
|
||||
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
||||
{
|
||||
bool foundRateLimitHeader = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
if (header.Name != null && header.Value != null)
|
||||
{
|
||||
Console.WriteLine($"{header.Name} - {header.Value}");
|
||||
var headerValue = header.Value?.ToString();
|
||||
if (!string.IsNullOrEmpty(headerValue))
|
||||
{
|
||||
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < retrySecs)
|
||||
myDeserializedClass.retryInSeconds = retrySecs;
|
||||
}
|
||||
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
||||
{
|
||||
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < resetValue)
|
||||
myDeserializedClass.retryInSeconds = resetValue;
|
||||
}
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
foundRateLimitHeader = true;
|
||||
}
|
||||
}
|
||||
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
||||
checkReset = true;
|
||||
else
|
||||
checkReset = false;
|
||||
}
|
||||
|
||||
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
||||
{
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
// A 2xx that will not parse is a genuine surprise; keep the detail for that case only.
|
||||
if (response.IsSuccessful)
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user