Changes Summary
appsettings.json — Added PoolEnabled to each API section: - TumblrApi → true - TumblrApi3 → false - TumblrApi4 → true DataAccess.cs — Added: - ApiKeyConfig class — holds credentials + metadata per key - ApiKeyPool class — manages pool with round-robin rotation, SQLite-backed state (ApiKeyPoolState, ApiKeyPoolMeta tables) - GetCurrentKey() — returns next key, skipping rate-limited ones, falls back to earliest-recovery if all are throttled - MarkRateLimited(key, retryUntil) / MarkAvailable(key) — persists state - Initialize() — discovers pool-enabled keys, detects single-key override mode - Refactored APIAccess.GrabNotes(), GrabPostWithReplies(), GrabLikes() to accept ApiKeyConfig param and log [Key#N] Program.cs — Updated: - Tracks apiExplicitlySet flag from -api/-api3/-api4 - Initializes ApiKeyPool at startup (pool mode or single-key override) - All 3 API callers updated to use pool rotation + 429 handling Startup Output - Pool mode: [Pool] Active keys: Key#1=TumblrApi, Key#2=TumblrApi4 - Single-key: [Pool] Single-key mode: TumblrApi3
This commit is contained in:
+417
-244
@@ -1547,51 +1547,267 @@ namespace URLNotesGrabberCORE
|
||||
#endregion Updates
|
||||
}
|
||||
|
||||
internal class APIAccess
|
||||
public class ApiKeyConfig
|
||||
{
|
||||
public static string Blog { get; set; } = string.Empty;
|
||||
private static IConfiguration Configuration { get; set; } = null!;
|
||||
private static string ApiConfigSection { get; set; } = "TumblrApi";
|
||||
public string SectionName { get; set; } = string.Empty;
|
||||
public int KeyNumber { get; set; }
|
||||
public string ConsumerKey { get; set; } = string.Empty;
|
||||
public string ConsumerSecret { get; set; } = string.Empty;
|
||||
public string OAuthToken { get; set; } = string.Empty;
|
||||
public string OAuthTokenSecret { get; set; } = string.Empty;
|
||||
public bool PoolEnabled { get; set; }
|
||||
}
|
||||
|
||||
static APIAccess()
|
||||
internal class ApiKeyPool
|
||||
{
|
||||
private static List<ApiKeyConfig> _keys = new();
|
||||
private static ApiKeyConfig? _overrideKey = null;
|
||||
private static bool _usePool = false;
|
||||
private static int _currentIndex = 0;
|
||||
private static string _dbPath = string.Empty;
|
||||
|
||||
public static bool IsPoolActive => _usePool;
|
||||
public static List<ApiKeyConfig> Keys => _keys;
|
||||
|
||||
public static void Initialize(IConfiguration config, string? dbPath, string? overrideSection = null)
|
||||
{
|
||||
Configuration = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
|
||||
.Build();
|
||||
_dbPath = dbPath ?? "..\\..\\..\\tl.db";
|
||||
EnsureStateTableExists();
|
||||
|
||||
ValidateApiSection(ApiConfigSection);
|
||||
}
|
||||
|
||||
public static void SetApiConfigSection(string sectionName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sectionName))
|
||||
sectionName = "TumblrApi";
|
||||
|
||||
ValidateApiSection(sectionName);
|
||||
ApiConfigSection = sectionName;
|
||||
Console.WriteLine($"Using API settings section: {ApiConfigSection}");
|
||||
}
|
||||
|
||||
private static void ValidateApiSection(string sectionName)
|
||||
{
|
||||
if (Configuration[$"{sectionName}:ConsumerKey"] == null ||
|
||||
Configuration[$"{sectionName}:ConsumerSecret"] == null ||
|
||||
Configuration[$"{sectionName}:OAuthToken"] == null ||
|
||||
Configuration[$"{sectionName}:OAuthTokenSecret"] == null)
|
||||
if (!string.IsNullOrEmpty(overrideSection))
|
||||
{
|
||||
throw new InvalidOperationException($"{sectionName} configuration is missing required keys in appsettings.json");
|
||||
var section = config.GetSection(overrideSection);
|
||||
if (section["ConsumerKey"] == null)
|
||||
throw new InvalidOperationException($"API section '{overrideSection}' not found in appsettings.json");
|
||||
|
||||
_overrideKey = new ApiKeyConfig
|
||||
{
|
||||
SectionName = overrideSection,
|
||||
KeyNumber = 1,
|
||||
ConsumerKey = section["ConsumerKey"]!,
|
||||
ConsumerSecret = section["ConsumerSecret"]!,
|
||||
OAuthToken = section["OAuthToken"]!,
|
||||
OAuthTokenSecret = section["OAuthTokenSecret"]!,
|
||||
PoolEnabled = true
|
||||
};
|
||||
_usePool = false;
|
||||
Console.WriteLine($"[Pool] Single-key mode: {overrideSection}");
|
||||
return;
|
||||
}
|
||||
|
||||
var root = config.AsEnumerable()
|
||||
.Where(kv => kv.Key.Contains(":ConsumerKey"))
|
||||
.Select(kv => kv.Key.Split(':')[0])
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
int keyNum = 1;
|
||||
foreach (var sectionName in root.OrderBy(s => s))
|
||||
{
|
||||
var section = config.GetSection(sectionName);
|
||||
var poolEnabledStr = section["PoolEnabled"];
|
||||
bool poolEnabled = !string.IsNullOrEmpty(poolEnabledStr) && bool.TryParse(poolEnabledStr, out var parsed) && parsed;
|
||||
|
||||
if (poolEnabled)
|
||||
{
|
||||
_keys.Add(new ApiKeyConfig
|
||||
{
|
||||
SectionName = sectionName,
|
||||
KeyNumber = keyNum,
|
||||
ConsumerKey = section["ConsumerKey"]!,
|
||||
ConsumerSecret = section["ConsumerSecret"]!,
|
||||
OAuthToken = section["OAuthToken"]!,
|
||||
OAuthTokenSecret = section["OAuthTokenSecret"]!,
|
||||
PoolEnabled = true
|
||||
});
|
||||
keyNum++;
|
||||
}
|
||||
}
|
||||
|
||||
if (_keys.Count == 0)
|
||||
{
|
||||
var fallback = config.GetSection("TumblrApi");
|
||||
if (fallback["ConsumerKey"] != null)
|
||||
{
|
||||
_keys.Add(new ApiKeyConfig
|
||||
{
|
||||
SectionName = "TumblrApi",
|
||||
KeyNumber = 1,
|
||||
ConsumerKey = fallback["ConsumerKey"]!,
|
||||
ConsumerSecret = fallback["ConsumerSecret"]!,
|
||||
OAuthToken = fallback["OAuthToken"]!,
|
||||
OAuthTokenSecret = fallback["OAuthTokenSecret"]!,
|
||||
PoolEnabled = true
|
||||
});
|
||||
Console.WriteLine("[Pool] No keys with PoolEnabled, falling back to TumblrApi");
|
||||
}
|
||||
}
|
||||
|
||||
_usePool = true;
|
||||
LoadState();
|
||||
|
||||
var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName}"));
|
||||
Console.WriteLine($"[Pool] Active keys: {mapping}");
|
||||
}
|
||||
|
||||
private static void EnsureStateTableExists()
|
||||
{
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
using var cmd1 = new System.Data.SQLite.SQLiteCommand(
|
||||
"CREATE TABLE IF NOT EXISTS ApiKeyPoolState (KeyName TEXT PRIMARY KEY, RetryUntil INTEGER DEFAULT 0)", conn);
|
||||
cmd1.ExecuteNonQuery();
|
||||
using var cmd2 = new System.Data.SQLite.SQLiteCommand(
|
||||
"CREATE TABLE IF NOT EXISTS ApiKeyPoolMeta (Id INTEGER PRIMARY KEY CHECK (Id = 1), LastIndex INTEGER DEFAULT 0)", conn);
|
||||
cmd2.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
private static void LoadState()
|
||||
{
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
|
||||
using var cmd = new System.Data.SQLite.SQLiteCommand("SELECT LastIndex FROM ApiKeyPoolMeta WHERE Id = 1", conn);
|
||||
var idx = cmd.ExecuteScalar();
|
||||
if (idx != null)
|
||||
_currentIndex = Convert.ToInt32(idx);
|
||||
|
||||
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
||||
foreach (var key in _keys)
|
||||
{
|
||||
using var readCmd = new System.Data.SQLite.SQLiteCommand(
|
||||
"SELECT RetryUntil FROM ApiKeyPoolState WHERE KeyName = @name", conn);
|
||||
readCmd.Parameters.AddWithValue("@name", key.SectionName);
|
||||
var retryUntil = readCmd.ExecuteScalar();
|
||||
if (retryUntil != null)
|
||||
{
|
||||
var retryTs = Convert.ToInt64(retryUntil);
|
||||
if (retryTs > now)
|
||||
{
|
||||
var remaining = retryTs - now;
|
||||
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, clears in {remaining}s");
|
||||
}
|
||||
else if (retryTs > 0)
|
||||
{
|
||||
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared on startup");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ConsumerKey => Configuration[$"{ApiConfigSection}:ConsumerKey"] ??
|
||||
throw new InvalidOperationException("ConsumerKey is not configured");
|
||||
private static string ConsumerSecret => Configuration[$"{ApiConfigSection}:ConsumerSecret"] ??
|
||||
throw new InvalidOperationException("ConsumerSecret is not configured");
|
||||
private static string OAuthToken => Configuration[$"{ApiConfigSection}:OAuthToken"] ??
|
||||
throw new InvalidOperationException("OAuthToken is not configured");
|
||||
private static string OAuthTokenSecret => Configuration[$"{ApiConfigSection}:OAuthTokenSecret"] ??
|
||||
throw new InvalidOperationException("OAuthTokenSecret is not configured");
|
||||
public static ApiKeyConfig GetCurrentKey()
|
||||
{
|
||||
if (!_usePool || _overrideKey != null)
|
||||
return _overrideKey!;
|
||||
|
||||
if (_keys.Count == 1)
|
||||
return _keys[0];
|
||||
|
||||
int attempts = 0;
|
||||
|
||||
while (attempts < _keys.Count)
|
||||
{
|
||||
var key = _keys[_currentIndex % _keys.Count];
|
||||
_currentIndex = (_currentIndex + 1) % _keys.Count;
|
||||
|
||||
if (IsKeyAvailable(key))
|
||||
{
|
||||
SaveState();
|
||||
return key;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
}
|
||||
|
||||
var earliest = _keys
|
||||
.Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) })
|
||||
.OrderBy(x => x.RetryUntil)
|
||||
.First();
|
||||
|
||||
Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)");
|
||||
|
||||
var chosen = earliest.Key;
|
||||
_currentIndex = (_keys.IndexOf(chosen) + 1) % _keys.Count;
|
||||
SaveState();
|
||||
return chosen;
|
||||
}
|
||||
|
||||
private static bool IsKeyAvailable(ApiKeyConfig key)
|
||||
{
|
||||
var retryUntil = GetRetryUntil(key);
|
||||
return retryUntil <= DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
||||
}
|
||||
|
||||
private static long GetRetryUntil(ApiKeyConfig key)
|
||||
{
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
||||
"SELECT RetryUntil FROM ApiKeyPoolState WHERE KeyName = @name", conn);
|
||||
cmd.Parameters.AddWithValue("@name", key.SectionName);
|
||||
var result = cmd.ExecuteScalar();
|
||||
return result != null ? Convert.ToInt64(result) : 0;
|
||||
}
|
||||
|
||||
public static void MarkRateLimited(ApiKeyConfig key, int retryInSeconds)
|
||||
{
|
||||
var retryUntil = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + retryInSeconds;
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
||||
"INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, @until)", conn);
|
||||
cmd.Parameters.AddWithValue("@name", key.SectionName);
|
||||
cmd.Parameters.AddWithValue("@until", retryUntil);
|
||||
cmd.ExecuteNonQuery();
|
||||
|
||||
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, retry in {retryInSeconds}s (until {DateTimeOffset.FromUnixTimeSeconds(retryUntil).LocalDateTime:HH:mm:ss})");
|
||||
}
|
||||
|
||||
public static void MarkAvailable(ApiKeyConfig key)
|
||||
{
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
||||
"INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, 0)", conn);
|
||||
cmd.Parameters.AddWithValue("@name", key.SectionName);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
private static void SaveState()
|
||||
{
|
||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||
conn.Open();
|
||||
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
||||
"INSERT OR REPLACE INTO ApiKeyPoolMeta (Id, LastIndex) VALUES (1, @idx)", conn);
|
||||
cmd.Parameters.AddWithValue("@idx", _currentIndex);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
internal class APIAccess
|
||||
{
|
||||
public static string Blog { get; set; } = string.Empty;
|
||||
|
||||
public static void SetApiConfigSection(string sectionName)
|
||||
{
|
||||
if (!ApiKeyPool.IsPoolActive)
|
||||
Console.WriteLine($"Using API settings section: {sectionName}");
|
||||
}
|
||||
|
||||
private static RestClient BuildClient(ApiKeyConfig key, string url)
|
||||
{
|
||||
var client = new RestClient(url);
|
||||
var oAuth1 = OAuth1Authenticator.ForAccessToken(
|
||||
consumerKey: key.ConsumerKey,
|
||||
consumerSecret: key.ConsumerSecret,
|
||||
token: key.OAuthToken,
|
||||
tokenSecret: key.OAuthTokenSecret,
|
||||
OAuthSignatureMethod.HmacSha1);
|
||||
client.Authenticator = oAuth1;
|
||||
return client;
|
||||
}
|
||||
|
||||
private static string FormatKeyLabel(ApiKeyConfig key) => $"[Key#{key.KeyNumber}]";
|
||||
|
||||
private static int GetRetryDelaySecondsFromHeaders(IEnumerable<HeaderParameter>? headers)
|
||||
{
|
||||
@@ -1652,7 +1868,7 @@ namespace URLNotesGrabberCORE
|
||||
return retryInSeconds > 0 ? retryInSeconds : 60;
|
||||
}
|
||||
|
||||
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
|
||||
public static async Task<Root> GrabNotes(ApiKeyConfig key, string blog, long ID, string? timestamp = null)
|
||||
{
|
||||
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all";
|
||||
URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
|
||||
@@ -1662,206 +1878,171 @@ namespace URLNotesGrabberCORE
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
// Create a new RestClient for each request to ensure fresh OAuth signatures
|
||||
using (var client = new RestClient(URL))
|
||||
using (var client = BuildClient(key, URL))
|
||||
{
|
||||
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
||||
consumerSecret: ConsumerSecret,
|
||||
token: OAuthToken,
|
||||
tokenSecret: OAuthTokenSecret,
|
||||
OAuthSignatureMethod.HmacSha1
|
||||
);
|
||||
|
||||
client.Authenticator = oAuth1;
|
||||
var request = new RestRequest(URL, Method.Get);
|
||||
var response = await client.ExecuteAsync(request);
|
||||
|
||||
var myJsonResponse = response.Content ?? string.Empty;
|
||||
Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
var myDeserializedClass = new Root();
|
||||
|
||||
try
|
||||
{
|
||||
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
||||
if (deserializedResult != null)
|
||||
{
|
||||
myDeserializedClass = deserializedResult;
|
||||
myDeserializedClass.rawJson = myJsonResponse;
|
||||
|
||||
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
|
||||
{
|
||||
myDeserializedClass.statusCode = "NotFound";
|
||||
}
|
||||
|
||||
// If the response JSON indicates a 429 (Too Many Requests) via meta.status or message,
|
||||
// treat it like a rate-limited response and attempt to read Retry headers.
|
||||
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
||||
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
||||
{
|
||||
// populate retryInSeconds from headers if possible
|
||||
if (response?.Headers != null)
|
||||
{
|
||||
bool checkResetLocal = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
string? headerName = header?.Name;
|
||||
string? headerValue = header?.Value?.ToString();
|
||||
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
||||
continue;
|
||||
|
||||
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
||||
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
||||
}
|
||||
|
||||
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
||||
}
|
||||
|
||||
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
||||
checkResetLocal = true;
|
||||
|
||||
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// surface rate-limit status back to caller
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Failed JSON: {myJsonResponse}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
|
||||
if (!response.IsSuccessful)
|
||||
{
|
||||
// Response.StatusCode may be null with some RestSharp responses.
|
||||
// Guard it and still attempt to extract retry time from headers (Retry-After, Remaining/Reset, X-RateLimit-Reset)
|
||||
string? statusStr = null;
|
||||
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
||||
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
||||
// Only set the status if we actually have one; otherwise leave existing value alone (may be null)
|
||||
if (!string.IsNullOrEmpty(statusStr))
|
||||
myDeserializedClass.statusCode = statusStr;
|
||||
|
||||
bool checkReset = false;
|
||||
|
||||
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
||||
{
|
||||
bool foundRateLimitHeader = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
if (header.Name != null && header.Value != null)
|
||||
{
|
||||
Console.WriteLine($"{header.Name} - {header.Value}");
|
||||
// Common header patterns used by APIs to indicate retry times:
|
||||
// - Retry-After: either seconds or HTTP date
|
||||
// - X-RateLimit-Reset: often seconds since epoch
|
||||
// - <something>Reset (after Remaining==0): seconds
|
||||
var headerValue = header.Value?.ToString();
|
||||
if (!string.IsNullOrEmpty(headerValue))
|
||||
{
|
||||
// 1) Retry-After header (seconds or date)
|
||||
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < retrySecs)
|
||||
myDeserializedClass.retryInSeconds = retrySecs;
|
||||
}
|
||||
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
||||
{
|
||||
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Common 'Reset' header: numeric seconds or epoch seconds
|
||||
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < resetValue)
|
||||
myDeserializedClass.retryInSeconds = resetValue;
|
||||
}
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
{
|
||||
// treat as epoch seconds -> compute secs until that epoch
|
||||
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
// 3) X-RateLimit-Reset header: often epoch seconds
|
||||
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
foundRateLimitHeader = true;
|
||||
}
|
||||
}
|
||||
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
||||
checkReset = true;
|
||||
else
|
||||
checkReset = false;
|
||||
}
|
||||
|
||||
// If we detected rate-limit related headers (Retry-After / X-RateLimit-Reset / Remaining/Reset)
|
||||
// or the HTTP status indicates 429, surface it.
|
||||
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
||||
{
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return myDeserializedClass;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<PostsRoot> GrabPostWithReplies(string blog, long postID, long timestamp)
|
||||
{
|
||||
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}";
|
||||
|
||||
// Create a new RestClient for each request to ensure fresh OAuth signatures
|
||||
using (var client = new RestClient(URL))
|
||||
{
|
||||
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
||||
consumerSecret: ConsumerSecret,
|
||||
token: OAuthToken,
|
||||
tokenSecret: OAuthTokenSecret,
|
||||
OAuthSignatureMethod.HmacSha1
|
||||
);
|
||||
|
||||
client.Authenticator = oAuth1;
|
||||
var request = new RestRequest(URL, Method.Get);
|
||||
var response = await client.ExecuteAsync(request);
|
||||
|
||||
var myJsonResponse = response.Content ?? string.Empty;
|
||||
Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
Console.WriteLine($"{FormatKeyLabel(key)} {timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
var myDeserializedClass = new Root();
|
||||
|
||||
try
|
||||
{
|
||||
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
||||
if (deserializedResult != null)
|
||||
{
|
||||
myDeserializedClass = deserializedResult;
|
||||
myDeserializedClass.rawJson = myJsonResponse;
|
||||
|
||||
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
|
||||
{
|
||||
myDeserializedClass.statusCode = "NotFound";
|
||||
}
|
||||
|
||||
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
||||
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
||||
{
|
||||
if (response?.Headers != null)
|
||||
{
|
||||
bool checkResetLocal = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
string? headerName = header?.Name;
|
||||
string? headerValue = header?.Value?.ToString();
|
||||
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
||||
continue;
|
||||
|
||||
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
||||
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
||||
}
|
||||
|
||||
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
||||
}
|
||||
|
||||
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
||||
checkResetLocal = true;
|
||||
|
||||
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Failed JSON: {myJsonResponse}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
|
||||
if (!response.IsSuccessful)
|
||||
{
|
||||
string? statusStr = null;
|
||||
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
||||
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
||||
if (!string.IsNullOrEmpty(statusStr))
|
||||
myDeserializedClass.statusCode = statusStr;
|
||||
|
||||
bool checkReset = false;
|
||||
|
||||
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
||||
{
|
||||
bool foundRateLimitHeader = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
if (header.Name != null && header.Value != null)
|
||||
{
|
||||
Console.WriteLine($"{header.Name} - {header.Value}");
|
||||
var headerValue = header.Value?.ToString();
|
||||
if (!string.IsNullOrEmpty(headerValue))
|
||||
{
|
||||
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < retrySecs)
|
||||
myDeserializedClass.retryInSeconds = retrySecs;
|
||||
}
|
||||
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
||||
{
|
||||
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < resetValue)
|
||||
myDeserializedClass.retryInSeconds = resetValue;
|
||||
}
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
foundRateLimitHeader = true;
|
||||
}
|
||||
}
|
||||
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
||||
checkReset = true;
|
||||
else
|
||||
checkReset = false;
|
||||
}
|
||||
|
||||
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
||||
{
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return myDeserializedClass;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<PostsRoot> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
|
||||
{
|
||||
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}";
|
||||
|
||||
using (var client = BuildClient(key, URL))
|
||||
{
|
||||
var request = new RestRequest(URL, Method.Get);
|
||||
var response = await client.ExecuteAsync(request);
|
||||
|
||||
var myJsonResponse = response.Content ?? string.Empty;
|
||||
Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
var myDeserializedClass = new PostsRoot();
|
||||
|
||||
// Check if response is successful and contains JSON
|
||||
@@ -1905,7 +2086,7 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<LikesRoot> GrabLikes(string blog, long beforeTimestamp = 0)
|
||||
public static async Task<LikesRoot> GrabLikes(ApiKeyConfig key, string blog, long beforeTimestamp = 0)
|
||||
{
|
||||
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/likes?npf=false&reblog_info=true";
|
||||
if (beforeTimestamp > 0)
|
||||
@@ -1913,21 +2094,13 @@ namespace URLNotesGrabberCORE
|
||||
URL += $"&before={beforeTimestamp}";
|
||||
}
|
||||
|
||||
using (var client = new RestClient(URL))
|
||||
using (var client = BuildClient(key, URL))
|
||||
{
|
||||
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
||||
consumerSecret: ConsumerSecret,
|
||||
token: OAuthToken,
|
||||
tokenSecret: OAuthTokenSecret,
|
||||
OAuthSignatureMethod.HmacSha1
|
||||
);
|
||||
|
||||
client.Authenticator = oAuth1;
|
||||
var request = new RestRequest(URL, Method.Get);
|
||||
var response = await client.ExecuteAsync(request);
|
||||
|
||||
var myJsonResponse = response.Content ?? string.Empty;
|
||||
Console.WriteLine($"[Likes API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
Console.WriteLine($"[Likes API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
||||
var myDeserializedClass = new LikesRoot();
|
||||
|
||||
// Check if response is successful and contains JSON
|
||||
|
||||
Reference in New Issue
Block a user