jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changes Summary

Browse Source 
appsettings.json — Added PoolEnabled to each API section:
- TumblrApi → true
- TumblrApi3 → false
- TumblrApi4 → true
DataAccess.cs — Added:
- ApiKeyConfig class — holds credentials + metadata per key
- ApiKeyPool class — manages pool with round-robin rotation, SQLite-backed state (ApiKeyPoolState, ApiKeyPoolMeta tables)
  - GetCurrentKey() — returns next key, skipping rate-limited ones, falls back to earliest-recovery if all are throttled
  - MarkRateLimited(key, retryUntil) / MarkAvailable(key) — persists state
  - Initialize() — discovers pool-enabled keys, detects single-key override mode
- Refactored APIAccess.GrabNotes(), GrabPostWithReplies(), GrabLikes() to accept ApiKeyConfig param and log [Key#N]
Program.cs — Updated:
- Tracks apiExplicitlySet flag from -api/-api3/-api4
- Initializes ApiKeyPool at startup (pool mode or single-key override)
- All 3 API callers updated to use pool rotation + 429 handling
Startup Output
- Pool mode: [Pool] Active keys: Key#1=TumblrApi, Key#2=TumblrApi4
- Single-key: [Pool] Single-key mode: TumblrApi3
This commit is contained in:
jim
2026-05-03 13:31:28 -05:00
parent de96c7c1c5
commit 6cec7afb53
3 changed files with 485 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+62 -23
View File
@@ -23,6 +23,7 @@ namespace URLNotesGrabberCORE
var settings = config.GetSection("appSettings");
string apiSectionName = "TumblrApi";
bool apiExplicitlySet = false;
string startFromBlogName = string.Empty;
List<string> filteredArgs = new List<string>();
for (int i = 0; i < args.Length; i++)
@@ -31,6 +32,7 @@ namespace URLNotesGrabberCORE
string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
{
apiSectionName = "TumblrApi3";
apiExplicitlySet = true;
continue;
}
@@ -38,6 +40,7 @@ namespace URLNotesGrabberCORE
string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase))
{
apiSectionName = "TumblrApi4";
apiExplicitlySet = true;
continue;
}
@@ -47,6 +50,7 @@ namespace URLNotesGrabberCORE
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
{
apiSectionName = args[i + 1].Trim();
apiExplicitlySet = true;
i++;
}
else
@@ -75,7 +79,12 @@ namespace URLNotesGrabberCORE
}
args = filteredArgs.ToArray();
APIAccess.SetApiConfigSection(apiSectionName);
string? dbPath = config["appSettings:PathDB"];
if (string.IsNullOrWhiteSpace(dbPath))
dbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "tl.db");
ApiKeyPool.Initialize(config, dbPath, apiExplicitlySet ? apiSectionName : null);
// Setup Dual Logging
bool enableFileLogging = settings.GetValue("EnableFileLogging", true);
@@ -439,9 +448,17 @@ namespace URLNotesGrabberCORE
try
{
Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
// Add 2-second delay before API call to avoid server-side rate limiting
await Task.Delay(2000);
var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp);
var key = ApiKeyPool.GetCurrentKey();
var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, timestamp);
if (postsResponse?.statusCode == "TooManyRequests")
{
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
Console.WriteLine($"[Reply Text] Rate limited, will retry with next key");
return;
}
if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
{
@@ -677,24 +694,16 @@ namespace URLNotesGrabberCORE
if (!lease.IsAcquired)
{
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return; // Might want to sleep instead, but this matches other functions
return;
}
var response = await APIAccess.GrabLikes(blogName, cursor);
var key = ApiKeyPool.GetCurrentKey();
var response = await APIAccess.GrabLikes(key, blogName, cursor);
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
if (response?.statusCode == "TooManyRequests")
{
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
// Set pulled = 1 to skip in future
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
break;
}
if (response == null || response.statusCode == "TooManyRequests")
{
int retry = response?.retryInSeconds ?? 60;
if (retry <= 0)
retry = 60;
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
int remaining = retry;
DateTime retryUntil = DateTime.Now.AddSeconds(retry);
@@ -705,7 +714,17 @@ namespace URLNotesGrabberCORE
Thread.Sleep(sleepSeconds * 1000);
remaining -= sleepSeconds;
}
continue; // Retry same cursor
continue;
}
if (response.meta?.status != 429)
ApiKeyPool.MarkAvailable(key);
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
{
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
break;
}
if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
@@ -879,16 +898,25 @@ namespace URLNotesGrabberCORE
int APICount = DataAccess.GetAPICount();
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
//Thread.Sleep(3000);
var allNotes = new List<dynamic>();
int page = 1;
string beforeTimestamp = post.Item3.ToString();
bool hasReplies = false;
const int maxPages = 500;
var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
var key = ApiKeyPool.GetCurrentKey();
var response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
if (response.statusCode == "TooManyRequests")
{
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
return "TooManyRequests";
}
if (response.meta?.status != 429)
ApiKeyPool.MarkAvailable(key);
// Handle 404 and error codes
if (IsNotFound(response))
{
Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}");
@@ -903,6 +931,8 @@ namespace URLNotesGrabberCORE
}
if (response.statusCode == "TooManyRequests")
{
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
for (int s = 0; s <= response.retryInSeconds; s += 60)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
@@ -959,8 +989,17 @@ namespace URLNotesGrabberCORE
break;
}
// Fetch next page
response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
key = ApiKeyPool.GetCurrentKey();
response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
if (response.statusCode == "TooManyRequests")
{
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
return "TooManyRequests";
}
if (response.meta?.status != 429)
ApiKeyPool.MarkAvailable(key);
if (IsNotFound(response))
{
Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}");