From 6cec7afb5346ecceee6122c383e9a8bbb6769eb5 Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 3 May 2026 13:31:28 -0500 Subject: [PATCH] =?UTF-8?q?Changes=20Summary=20appsettings.json=20?= =?UTF-8?q?=E2=80=94=20Added=20PoolEnabled=20to=20each=20API=20section:=20?= =?UTF-8?q?-=20TumblrApi=20=E2=86=92=20true=20-=20TumblrApi3=20=E2=86=92?= =?UTF-8?q?=20false=20-=20TumblrApi4=20=E2=86=92=20true=20DataAccess.cs=20?= =?UTF-8?q?=E2=80=94=20Added:=20-=20ApiKeyConfig=20class=20=E2=80=94=20hol?= =?UTF-8?q?ds=20credentials=20+=20metadata=20per=20key=20-=20ApiKeyPool=20?= =?UTF-8?q?class=20=E2=80=94=20manages=20pool=20with=20round-robin=20rotat?= =?UTF-8?q?ion,=20SQLite-backed=20state=20(ApiKeyPoolState,=20ApiKeyPoolMe?= =?UTF-8?q?ta=20tables)=20=20=20-=20GetCurrentKey()=20=E2=80=94=20returns?= =?UTF-8?q?=20next=20key,=20skipping=20rate-limited=20ones,=20falls=20back?= =?UTF-8?q?=20to=20earliest-recovery=20if=20all=20are=20throttled=20=20=20?= =?UTF-8?q?-=20MarkRateLimited(key,=20retryUntil)=20/=20MarkAvailable(key)?= =?UTF-8?q?=20=E2=80=94=20persists=20state=20=20=20-=20Initialize()=20?= =?UTF-8?q?=E2=80=94=20discovers=20pool-enabled=20keys,=20detects=20single?= =?UTF-8?q?-key=20override=20mode=20-=20Refactored=20APIAccess.GrabNotes()?= =?UTF-8?q?,=20GrabPostWithReplies(),=20GrabLikes()=20to=20accept=20ApiKey?= =?UTF-8?q?Config=20param=20and=20log=20[Key#N]=20Program.cs=20=E2=80=94?= =?UTF-8?q?=20Updated:=20-=20Tracks=20apiExplicitlySet=20flag=20from=20-ap?= =?UTF-8?q?i/-api3/-api4=20-=20Initializes=20ApiKeyPool=20at=20startup=20(?= =?UTF-8?q?pool=20mode=20or=20single-key=20override)=20-=20All=203=20API?= =?UTF-8?q?=20callers=20updated=20to=20use=20pool=20rotation=20+=20429=20h?= =?UTF-8?q?andling=20Startup=20Output=20-=20Pool=20mode:=20[Pool]=20Active?= =?UTF-8?q?=20keys:=20Key#1=3DTumblrApi,=20Key#2=3DTumblrApi4=20-=20Single?= =?UTF-8?q?-key:=20[Pool]=20Single-key=20mode:=20TumblrApi3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- URLNotesGrabberCORE/DataAccess.cs | 661 +++++++++++++++++---------- URLNotesGrabberCORE/Program.cs | 85 +++- URLNotesGrabberCORE/appsettings.json | 9 +- 3 files changed, 485 insertions(+), 270 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 4c33938..aa6b136 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -1547,51 +1547,267 @@ namespace URLNotesGrabberCORE #endregion Updates } - internal class APIAccess + public class ApiKeyConfig { - public static string Blog { get; set; } = string.Empty; - private static IConfiguration Configuration { get; set; } = null!; - private static string ApiConfigSection { get; set; } = "TumblrApi"; + public string SectionName { get; set; } = string.Empty; + public int KeyNumber { get; set; } + public string ConsumerKey { get; set; } = string.Empty; + public string ConsumerSecret { get; set; } = string.Empty; + public string OAuthToken { get; set; } = string.Empty; + public string OAuthTokenSecret { get; set; } = string.Empty; + public bool PoolEnabled { get; set; } + } - static APIAccess() + internal class ApiKeyPool + { + private static List _keys = new(); + private static ApiKeyConfig? _overrideKey = null; + private static bool _usePool = false; + private static int _currentIndex = 0; + private static string _dbPath = string.Empty; + + public static bool IsPoolActive => _usePool; + public static List Keys => _keys; + + public static void Initialize(IConfiguration config, string? dbPath, string? overrideSection = null) { - Configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .Build(); + _dbPath = dbPath ?? "..\\..\\..\\tl.db"; + EnsureStateTableExists(); - ValidateApiSection(ApiConfigSection); - } - - public static void SetApiConfigSection(string sectionName) - { - if (string.IsNullOrWhiteSpace(sectionName)) - sectionName = "TumblrApi"; - - ValidateApiSection(sectionName); - ApiConfigSection = sectionName; - Console.WriteLine($"Using API settings section: {ApiConfigSection}"); - } - - private static void ValidateApiSection(string sectionName) - { - if (Configuration[$"{sectionName}:ConsumerKey"] == null || - Configuration[$"{sectionName}:ConsumerSecret"] == null || - Configuration[$"{sectionName}:OAuthToken"] == null || - Configuration[$"{sectionName}:OAuthTokenSecret"] == null) + if (!string.IsNullOrEmpty(overrideSection)) { - throw new InvalidOperationException($"{sectionName} configuration is missing required keys in appsettings.json"); + var section = config.GetSection(overrideSection); + if (section["ConsumerKey"] == null) + throw new InvalidOperationException($"API section '{overrideSection}' not found in appsettings.json"); + + _overrideKey = new ApiKeyConfig + { + SectionName = overrideSection, + KeyNumber = 1, + ConsumerKey = section["ConsumerKey"]!, + ConsumerSecret = section["ConsumerSecret"]!, + OAuthToken = section["OAuthToken"]!, + OAuthTokenSecret = section["OAuthTokenSecret"]!, + PoolEnabled = true + }; + _usePool = false; + Console.WriteLine($"[Pool] Single-key mode: {overrideSection}"); + return; + } + + var root = config.AsEnumerable() + .Where(kv => kv.Key.Contains(":ConsumerKey")) + .Select(kv => kv.Key.Split(':')[0]) + .Distinct() + .ToList(); + + int keyNum = 1; + foreach (var sectionName in root.OrderBy(s => s)) + { + var section = config.GetSection(sectionName); + var poolEnabledStr = section["PoolEnabled"]; + bool poolEnabled = !string.IsNullOrEmpty(poolEnabledStr) && bool.TryParse(poolEnabledStr, out var parsed) && parsed; + + if (poolEnabled) + { + _keys.Add(new ApiKeyConfig + { + SectionName = sectionName, + KeyNumber = keyNum, + ConsumerKey = section["ConsumerKey"]!, + ConsumerSecret = section["ConsumerSecret"]!, + OAuthToken = section["OAuthToken"]!, + OAuthTokenSecret = section["OAuthTokenSecret"]!, + PoolEnabled = true + }); + keyNum++; + } + } + + if (_keys.Count == 0) + { + var fallback = config.GetSection("TumblrApi"); + if (fallback["ConsumerKey"] != null) + { + _keys.Add(new ApiKeyConfig + { + SectionName = "TumblrApi", + KeyNumber = 1, + ConsumerKey = fallback["ConsumerKey"]!, + ConsumerSecret = fallback["ConsumerSecret"]!, + OAuthToken = fallback["OAuthToken"]!, + OAuthTokenSecret = fallback["OAuthTokenSecret"]!, + PoolEnabled = true + }); + Console.WriteLine("[Pool] No keys with PoolEnabled, falling back to TumblrApi"); + } + } + + _usePool = true; + LoadState(); + + var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName}")); + Console.WriteLine($"[Pool] Active keys: {mapping}"); + } + + private static void EnsureStateTableExists() + { + using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); + conn.Open(); + using var cmd1 = new System.Data.SQLite.SQLiteCommand( + "CREATE TABLE IF NOT EXISTS ApiKeyPoolState (KeyName TEXT PRIMARY KEY, RetryUntil INTEGER DEFAULT 0)", conn); + cmd1.ExecuteNonQuery(); + using var cmd2 = new System.Data.SQLite.SQLiteCommand( + "CREATE TABLE IF NOT EXISTS ApiKeyPoolMeta (Id INTEGER PRIMARY KEY CHECK (Id = 1), LastIndex INTEGER DEFAULT 0)", conn); + cmd2.ExecuteNonQuery(); + } + + private static void LoadState() + { + using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); + conn.Open(); + + using var cmd = new System.Data.SQLite.SQLiteCommand("SELECT LastIndex FROM ApiKeyPoolMeta WHERE Id = 1", conn); + var idx = cmd.ExecuteScalar(); + if (idx != null) + _currentIndex = Convert.ToInt32(idx); + + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + foreach (var key in _keys) + { + using var readCmd = new System.Data.SQLite.SQLiteCommand( + "SELECT RetryUntil FROM ApiKeyPoolState WHERE KeyName = @name", conn); + readCmd.Parameters.AddWithValue("@name", key.SectionName); + var retryUntil = readCmd.ExecuteScalar(); + if (retryUntil != null) + { + var retryTs = Convert.ToInt64(retryUntil); + if (retryTs > now) + { + var remaining = retryTs - now; + Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, clears in {remaining}s"); + } + else if (retryTs > 0) + { + Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared on startup"); + } + } } } - private static string ConsumerKey => Configuration[$"{ApiConfigSection}:ConsumerKey"] ?? - throw new InvalidOperationException("ConsumerKey is not configured"); - private static string ConsumerSecret => Configuration[$"{ApiConfigSection}:ConsumerSecret"] ?? - throw new InvalidOperationException("ConsumerSecret is not configured"); - private static string OAuthToken => Configuration[$"{ApiConfigSection}:OAuthToken"] ?? - throw new InvalidOperationException("OAuthToken is not configured"); - private static string OAuthTokenSecret => Configuration[$"{ApiConfigSection}:OAuthTokenSecret"] ?? - throw new InvalidOperationException("OAuthTokenSecret is not configured"); + public static ApiKeyConfig GetCurrentKey() + { + if (!_usePool || _overrideKey != null) + return _overrideKey!; + + if (_keys.Count == 1) + return _keys[0]; + + int attempts = 0; + + while (attempts < _keys.Count) + { + var key = _keys[_currentIndex % _keys.Count]; + _currentIndex = (_currentIndex + 1) % _keys.Count; + + if (IsKeyAvailable(key)) + { + SaveState(); + return key; + } + + attempts++; + } + + var earliest = _keys + .Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) }) + .OrderBy(x => x.RetryUntil) + .First(); + + Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)"); + + var chosen = earliest.Key; + _currentIndex = (_keys.IndexOf(chosen) + 1) % _keys.Count; + SaveState(); + return chosen; + } + + private static bool IsKeyAvailable(ApiKeyConfig key) + { + var retryUntil = GetRetryUntil(key); + return retryUntil <= DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + } + + private static long GetRetryUntil(ApiKeyConfig key) + { + using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); + conn.Open(); + using var cmd = new System.Data.SQLite.SQLiteCommand( + "SELECT RetryUntil FROM ApiKeyPoolState WHERE KeyName = @name", conn); + cmd.Parameters.AddWithValue("@name", key.SectionName); + var result = cmd.ExecuteScalar(); + return result != null ? Convert.ToInt64(result) : 0; + } + + public static void MarkRateLimited(ApiKeyConfig key, int retryInSeconds) + { + var retryUntil = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + retryInSeconds; + using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); + conn.Open(); + using var cmd = new System.Data.SQLite.SQLiteCommand( + "INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, @until)", conn); + cmd.Parameters.AddWithValue("@name", key.SectionName); + cmd.Parameters.AddWithValue("@until", retryUntil); + cmd.ExecuteNonQuery(); + + Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, retry in {retryInSeconds}s (until {DateTimeOffset.FromUnixTimeSeconds(retryUntil).LocalDateTime:HH:mm:ss})"); + } + + public static void MarkAvailable(ApiKeyConfig key) + { + using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); + conn.Open(); + using var cmd = new System.Data.SQLite.SQLiteCommand( + "INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, 0)", conn); + cmd.Parameters.AddWithValue("@name", key.SectionName); + cmd.ExecuteNonQuery(); + } + + private static void SaveState() + { + using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); + conn.Open(); + using var cmd = new System.Data.SQLite.SQLiteCommand( + "INSERT OR REPLACE INTO ApiKeyPoolMeta (Id, LastIndex) VALUES (1, @idx)", conn); + cmd.Parameters.AddWithValue("@idx", _currentIndex); + cmd.ExecuteNonQuery(); + } + } + + internal class APIAccess + { + public static string Blog { get; set; } = string.Empty; + + public static void SetApiConfigSection(string sectionName) + { + if (!ApiKeyPool.IsPoolActive) + Console.WriteLine($"Using API settings section: {sectionName}"); + } + + private static RestClient BuildClient(ApiKeyConfig key, string url) + { + var client = new RestClient(url); + var oAuth1 = OAuth1Authenticator.ForAccessToken( + consumerKey: key.ConsumerKey, + consumerSecret: key.ConsumerSecret, + token: key.OAuthToken, + tokenSecret: key.OAuthTokenSecret, + OAuthSignatureMethod.HmacSha1); + client.Authenticator = oAuth1; + return client; + } + + private static string FormatKeyLabel(ApiKeyConfig key) => $"[Key#{key.KeyNumber}]"; private static int GetRetryDelaySecondsFromHeaders(IEnumerable? headers) { @@ -1652,7 +1868,7 @@ namespace URLNotesGrabberCORE return retryInSeconds > 0 ? retryInSeconds : 60; } - public static async Task GrabNotes(string blog, long ID, string? timestamp = null) + public static async Task GrabNotes(ApiKeyConfig key, string blog, long ID, string? timestamp = null) { var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all"; URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString()); @@ -1662,206 +1878,171 @@ namespace URLNotesGrabberCORE await Task.Delay(100); } - // Create a new RestClient for each request to ensure fresh OAuth signatures - using (var client = new RestClient(URL)) + using (var client = BuildClient(key, URL)) { - var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, - consumerSecret: ConsumerSecret, - token: OAuthToken, - tokenSecret: OAuthTokenSecret, - OAuthSignatureMethod.HmacSha1 - ); - - client.Authenticator = oAuth1; - var request = new RestRequest(URL, Method.Get); - var response = await client.ExecuteAsync(request); - - var myJsonResponse = response.Content ?? string.Empty; - Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}"); - var myDeserializedClass = new Root(); - - try - { - var deserializedResult = JsonConvert.DeserializeObject(myJsonResponse); - if (deserializedResult != null) - { - myDeserializedClass = deserializedResult; - myDeserializedClass.rawJson = myJsonResponse; - - if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404) - { - myDeserializedClass.statusCode = "NotFound"; - } - - // If the response JSON indicates a 429 (Too Many Requests) via meta.status or message, - // treat it like a rate-limited response and attempt to read Retry headers. - bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429; - bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0; - - if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))) - { - // populate retryInSeconds from headers if possible - if (response?.Headers != null) - { - bool checkResetLocal = false; - foreach (var header in response.Headers) - { - string? headerName = header?.Name; - string? headerValue = header?.Value?.ToString(); - if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue)) - continue; - - if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase)) - { - if (int.TryParse(headerValue, out int retrySecs)) - myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs); - else if (DateTimeOffset.TryParse(headerValue, out var dto)) - myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds)); - } - - if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch)) - { - var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); - myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs); - } - - if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0") - checkResetLocal = true; - - if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0) - { - if (int.TryParse(headerValue, out int resetValue)) - myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue); - else if (long.TryParse(headerValue, out long epochVal)) - myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds())); - } - } - } - - // surface rate-limit status back to caller - myDeserializedClass.statusCode = "TooManyRequests"; - } - } - } - catch (Exception ex) - { - Console.WriteLine($"Failed JSON: {myJsonResponse}"); - Console.WriteLine(ex.ToString()); - - if (!response.IsSuccessful) - { - // Response.StatusCode may be null with some RestSharp responses. - // Guard it and still attempt to extract retry time from headers (Retry-After, Remaining/Reset, X-RateLimit-Reset) - string? statusStr = null; - try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; } - Console.WriteLine($"{statusStr}\t{response?.StatusDescription}"); - // Only set the status if we actually have one; otherwise leave existing value alone (may be null) - if (!string.IsNullOrEmpty(statusStr)) - myDeserializedClass.statusCode = statusStr; - - bool checkReset = false; - - if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null) - { - bool foundRateLimitHeader = false; - foreach (var header in response.Headers) - { - if (header.Name != null && header.Value != null) - { - Console.WriteLine($"{header.Name} - {header.Value}"); - // Common header patterns used by APIs to indicate retry times: - // - Retry-After: either seconds or HTTP date - // - X-RateLimit-Reset: often seconds since epoch - // - Reset (after Remaining==0): seconds - var headerValue = header.Value?.ToString(); - if (!string.IsNullOrEmpty(headerValue)) - { - // 1) Retry-After header (seconds or date) - if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase)) - { - if (int.TryParse(headerValue, out int retrySecs)) - { - if (myDeserializedClass.retryInSeconds < retrySecs) - myDeserializedClass.retryInSeconds = retrySecs; - } - else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto)) - { - var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds); - if (myDeserializedClass.retryInSeconds < secs) - myDeserializedClass.retryInSeconds = secs; - } - } - - // 2) Common 'Reset' header: numeric seconds or epoch seconds - if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase)) - { - if (int.TryParse(headerValue, out int resetValue)) - { - if (myDeserializedClass.retryInSeconds < resetValue) - myDeserializedClass.retryInSeconds = resetValue; - } - else if (long.TryParse(headerValue, out long epochVal)) - { - // treat as epoch seconds -> compute secs until that epoch - var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); - if (myDeserializedClass.retryInSeconds < secs) - myDeserializedClass.retryInSeconds = secs; - } - } - - // 3) X-RateLimit-Reset header: often epoch seconds - if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0) - { - if (long.TryParse(headerValue, out long epoch)) - { - var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); - if (myDeserializedClass.retryInSeconds < secs) - myDeserializedClass.retryInSeconds = secs; - } - foundRateLimitHeader = true; - } - } - if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0") - checkReset = true; - else - checkReset = false; - } - - // If we detected rate-limit related headers (Retry-After / X-RateLimit-Reset / Remaining/Reset) - // or the HTTP status indicates 429, surface it. - if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)) - { - myDeserializedClass.statusCode = "TooManyRequests"; - } - } - } - } - } - - return myDeserializedClass; - } - } - - public static async Task GrabPostWithReplies(string blog, long postID, long timestamp) - { - var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}"; - - // Create a new RestClient for each request to ensure fresh OAuth signatures - using (var client = new RestClient(URL)) - { - var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, - consumerSecret: ConsumerSecret, - token: OAuthToken, - tokenSecret: OAuthTokenSecret, - OAuthSignatureMethod.HmacSha1 - ); - - client.Authenticator = oAuth1; var request = new RestRequest(URL, Method.Get); var response = await client.ExecuteAsync(request); var myJsonResponse = response.Content ?? string.Empty; - Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}"); + Console.WriteLine($"{FormatKeyLabel(key)} {timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}"); + var myDeserializedClass = new Root(); + + try + { + var deserializedResult = JsonConvert.DeserializeObject(myJsonResponse); + if (deserializedResult != null) + { + myDeserializedClass = deserializedResult; + myDeserializedClass.rawJson = myJsonResponse; + + if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404) + { + myDeserializedClass.statusCode = "NotFound"; + } + + bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429; + bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0; + + if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))) + { + if (response?.Headers != null) + { + bool checkResetLocal = false; + foreach (var header in response.Headers) + { + string? headerName = header?.Name; + string? headerValue = header?.Value?.ToString(); + if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue)) + continue; + + if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int retrySecs)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs); + else if (DateTimeOffset.TryParse(headerValue, out var dto)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds)); + } + + if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch)) + { + var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs); + } + + if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0") + checkResetLocal = true; + + if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0) + { + if (int.TryParse(headerValue, out int resetValue)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue); + else if (long.TryParse(headerValue, out long epochVal)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds())); + } + } + } + + myDeserializedClass.statusCode = "TooManyRequests"; + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed JSON: {myJsonResponse}"); + Console.WriteLine(ex.ToString()); + + if (!response.IsSuccessful) + { + string? statusStr = null; + try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; } + Console.WriteLine($"{statusStr}\t{response?.StatusDescription}"); + if (!string.IsNullOrEmpty(statusStr)) + myDeserializedClass.statusCode = statusStr; + + bool checkReset = false; + + if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null) + { + bool foundRateLimitHeader = false; + foreach (var header in response.Headers) + { + if (header.Name != null && header.Value != null) + { + Console.WriteLine($"{header.Name} - {header.Value}"); + var headerValue = header.Value?.ToString(); + if (!string.IsNullOrEmpty(headerValue)) + { + if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int retrySecs)) + { + if (myDeserializedClass.retryInSeconds < retrySecs) + myDeserializedClass.retryInSeconds = retrySecs; + } + else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto)) + { + var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + } + + if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int resetValue)) + { + if (myDeserializedClass.retryInSeconds < resetValue) + myDeserializedClass.retryInSeconds = resetValue; + } + else if (long.TryParse(headerValue, out long epochVal)) + { + var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + } + + if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0) + { + if (long.TryParse(headerValue, out long epoch)) + { + var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + foundRateLimitHeader = true; + } + } + if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0") + checkReset = true; + else + checkReset = false; + } + + if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)) + { + myDeserializedClass.statusCode = "TooManyRequests"; + } + } + } + } + } + + return myDeserializedClass; + } + } + + public static async Task GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp) + { + var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}"; + + using (var client = BuildClient(key, URL)) + { + var request = new RestRequest(URL, Method.Get); + var response = await client.ExecuteAsync(request); + + var myJsonResponse = response.Content ?? string.Empty; + Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}"); var myDeserializedClass = new PostsRoot(); // Check if response is successful and contains JSON @@ -1905,7 +2086,7 @@ namespace URLNotesGrabberCORE } } - public static async Task GrabLikes(string blog, long beforeTimestamp = 0) + public static async Task GrabLikes(ApiKeyConfig key, string blog, long beforeTimestamp = 0) { var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/likes?npf=false&reblog_info=true"; if (beforeTimestamp > 0) @@ -1913,21 +2094,13 @@ namespace URLNotesGrabberCORE URL += $"&before={beforeTimestamp}"; } - using (var client = new RestClient(URL)) + using (var client = BuildClient(key, URL)) { - var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, - consumerSecret: ConsumerSecret, - token: OAuthToken, - tokenSecret: OAuthTokenSecret, - OAuthSignatureMethod.HmacSha1 - ); - - client.Authenticator = oAuth1; var request = new RestRequest(URL, Method.Get); var response = await client.ExecuteAsync(request); var myJsonResponse = response.Content ?? string.Empty; - Console.WriteLine($"[Likes API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}"); + Console.WriteLine($"[Likes API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}"); var myDeserializedClass = new LikesRoot(); // Check if response is successful and contains JSON diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 86b1a69..f5e7bfb 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -23,6 +23,7 @@ namespace URLNotesGrabberCORE var settings = config.GetSection("appSettings"); string apiSectionName = "TumblrApi"; + bool apiExplicitlySet = false; string startFromBlogName = string.Empty; List filteredArgs = new List(); for (int i = 0; i < args.Length; i++) @@ -31,6 +32,7 @@ namespace URLNotesGrabberCORE string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase)) { apiSectionName = "TumblrApi3"; + apiExplicitlySet = true; continue; } @@ -38,6 +40,7 @@ namespace URLNotesGrabberCORE string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase)) { apiSectionName = "TumblrApi4"; + apiExplicitlySet = true; continue; } @@ -47,6 +50,7 @@ namespace URLNotesGrabberCORE if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1])) { apiSectionName = args[i + 1].Trim(); + apiExplicitlySet = true; i++; } else @@ -75,7 +79,12 @@ namespace URLNotesGrabberCORE } args = filteredArgs.ToArray(); - APIAccess.SetApiConfigSection(apiSectionName); + + string? dbPath = config["appSettings:PathDB"]; + if (string.IsNullOrWhiteSpace(dbPath)) + dbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "tl.db"); + + ApiKeyPool.Initialize(config, dbPath, apiExplicitlySet ? apiSectionName : null); // Setup Dual Logging bool enableFileLogging = settings.GetValue("EnableFileLogging", true); @@ -439,9 +448,17 @@ namespace URLNotesGrabberCORE try { Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}"); - // Add 2-second delay before API call to avoid server-side rate limiting await Task.Delay(2000); - var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp); + var key = ApiKeyPool.GetCurrentKey(); + var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp); + + if (postsResponse?.statusCode == "TooManyRequests") + { + int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60; + ApiKeyPool.MarkRateLimited(key, retry); + Console.WriteLine($"[Reply Text] Rate limited, will retry with next key"); + return; + } if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0) { @@ -677,24 +694,16 @@ namespace URLNotesGrabberCORE if (!lease.IsAcquired) { Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); - return; // Might want to sleep instead, but this matches other functions + return; } - var response = await APIAccess.GrabLikes(blogName, cursor); + var key = ApiKeyPool.GetCurrentKey(); + var response = await APIAccess.GrabLikes(key, blogName, cursor); - if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404)) + if (response?.statusCode == "TooManyRequests") { - Console.WriteLine($"API returned 404 Not Found for {blogName} Likes"); - // Set pulled = 1 to skip in future - DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); - break; - } - - if (response == null || response.statusCode == "TooManyRequests") - { - int retry = response?.retryInSeconds ?? 60; - if (retry <= 0) - retry = 60; + int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60; + ApiKeyPool.MarkRateLimited(key, retry); int remaining = retry; DateTime retryUntil = DateTime.Now.AddSeconds(retry); @@ -705,7 +714,17 @@ namespace URLNotesGrabberCORE Thread.Sleep(sleepSeconds * 1000); remaining -= sleepSeconds; } - continue; // Retry same cursor + continue; + } + + if (response.meta?.status != 429) + ApiKeyPool.MarkAvailable(key); + + if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404)) + { + Console.WriteLine($"API returned 404 Not Found for {blogName} Likes"); + DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); + break; } if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0) @@ -879,16 +898,25 @@ namespace URLNotesGrabberCORE int APICount = DataAccess.GetAPICount(); Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}"); - //Thread.Sleep(3000); var allNotes = new List(); int page = 1; string beforeTimestamp = post.Item3.ToString(); bool hasReplies = false; const int maxPages = 500; - var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult(); + var key = ApiKeyPool.GetCurrentKey(); + var response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp); + + if (response.statusCode == "TooManyRequests") + { + int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60; + ApiKeyPool.MarkRateLimited(key, retry); + return "TooManyRequests"; + } + + if (response.meta?.status != 429) + ApiKeyPool.MarkAvailable(key); - // Handle 404 and error codes if (IsNotFound(response)) { Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}"); @@ -903,6 +931,8 @@ namespace URLNotesGrabberCORE } if (response.statusCode == "TooManyRequests") { + int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60; + ApiKeyPool.MarkRateLimited(key, retry); for (int s = 0; s <= response.retryInSeconds; s += 60) { Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString()); @@ -959,8 +989,17 @@ namespace URLNotesGrabberCORE break; } - // Fetch next page - response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult(); + key = ApiKeyPool.GetCurrentKey(); + response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp); + if (response.statusCode == "TooManyRequests") + { + int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60; + ApiKeyPool.MarkRateLimited(key, retry); + return "TooManyRequests"; + } + if (response.meta?.status != 429) + ApiKeyPool.MarkAvailable(key); + if (IsNotFound(response)) { Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}"); diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index e3ea0e9..0509c0d 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -17,18 +17,21 @@ "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3", "ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG", "OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J", - "OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w" + "OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w", + "PoolEnabled": true }, "TumblrApi3": { "ConsumerKey": "Jmoh13AS9hKBYsf939uENQsiWUuZJLFT3do4YK9P0bpspVuxkH", "ConsumerSecret": "8LcHjuqpOS9gZZPaML1joT330w5NqyGdreSqSBazdAWCADzcms", "OAuthToken": "Gm2esFTeb5LKsb4A2YJGrF2Udycfc0AF8afof2qNGyzNSq1qWM", - "OAuthTokenSecret": "Kfokor80s5fVue91OmNHidCdILZ9AnHtRplrct2P6Q75jgNTLa" + "OAuthTokenSecret": "Kfokor80s5fVue91OmNHidCdILZ9AnHtRplrct2P6Q75jgNTLa", + "PoolEnabled": false }, "TumblrApi4": { "ConsumerKey": "gOuydIEENkRmEvvuf57R69yPRb39FC0Egb9p6ntxWuCuFzlsV3", "ConsumerSecret": "0my913slXAHgra4mEEYV98stJ3wXGjPTWJeviiZXwJ2AqzYtIF ", "OAuthToken": "JHk08YufSrsetTR1Ekuh10cPOzC6rjqU5WwVu3uV4vvLzp0w8w", - "OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb" + "OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb", + "PoolEnabled": true } } \ No newline at end of file