jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Correct CollectMissingReplyText to use API key rate limiting

Browse Source
This commit is contained in:
jim
2026-05-06 09:16:36 -05:00
parent 9c77ecce4b
commit 1507086269
3 changed files with 30 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+23 -37
View File
@@ -460,7 +460,7 @@ namespace URLNotesGrabberCORE
{
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
Console.WriteLine($"[Reply Text] Rate limited, will retry with next key");
Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
return;
}
@@ -559,62 +559,48 @@ namespace URLNotesGrabberCORE
DataAccess.EnsureReplyTextColumnExists();
Console.WriteLine("Starting collection of missing reply text...");
Console.WriteLine("Processing 10 posts at a time.");
Console.WriteLine("Processing replies directly from Tumblr API.");
Console.WriteLine();
int totalProcessedCount = 0;
int successCount = 0;
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{
PermitLimit = 300,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
QueueLimit = 1,
Window = TimeSpan.FromMinutes(1),
SegmentsPerWindow = 60,
AutoReplenishment = true
});
// Get posts with missing reply text
while (true)
{
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText();
if (postsWithFilledReplies.Count == 0)
var batch = DataAccess.GetRepliesWithFilledText();
if (batch is null || batch.Count == 0)
{
Console.WriteLine("No more posts with filled reply text found. Collection complete.");
Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
break;
}
Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
int batchProcessedCount = 0;
Console.WriteLine($"[Reply Text] Found {batch.Count} replies to process");
foreach (var post in postsWithFilledReplies)
foreach (var reply in batch)
{
using RateLimitLease lease = limiter.AttemptAcquire(1);
if (lease.IsAcquired)
{
Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
batchProcessedCount++;
totalProcessedCount++;
var blogName = (reply as Tuple<string, long, long>).Item1;
var postID = (reply as Tuple<string, long, long>).Item2;
var timestamp = (reply as Tuple<string, long, long>).Item3;
Console.WriteLine($"[Reply Text] Processing {blogName}/{postID}/{timestamp}");
// Add 1 second delay between attempts
await Task.Delay(1000);
}
else
await FetchAndStoreReplyText(blogName, postID, timestamp);
totalProcessedCount++;
if (totalProcessedCount % 10 == 0)
{
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
return;
await Task.Delay(2000);
}
}
Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
}
Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
Console.WriteLine($"[Reply Text] Completed. Total processed: {totalProcessedCount} posts.");
}
catch (Exception ex)
{
Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
Console.WriteLine($"[Reply Text] Error: {ex.Message}");
Console.WriteLine(ex.ToString());
}
}