fix: skip API calls when all keys are rate-limited in -collect mode
Pre-flight check on the CollectNotes loop now sleeps until at least one key recovers instead of issuing a wasted 429-bound request per iteration. Extracts the countdown into ApiKeyPool.SleepUntilAnyAvailable (30s refresh) and reuses it in CollectLikes and GrabNotes. Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
@@ -1948,6 +1948,28 @@ namespace URLNotesGrabberCORE
|
|||||||
return all;
|
return all;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void SleepUntilAnyAvailable(int refreshSeconds = 30)
|
||||||
|
{
|
||||||
|
if (refreshSeconds <= 0) refreshSeconds = 30;
|
||||||
|
if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return;
|
||||||
|
|
||||||
|
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
|
||||||
|
int remaining = minRetry;
|
||||||
|
while (remaining > 0)
|
||||||
|
{
|
||||||
|
Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T"));
|
||||||
|
int sleepSeconds = Math.Min(refreshSeconds, remaining);
|
||||||
|
Thread.Sleep(sleepSeconds * 1000);
|
||||||
|
remaining -= sleepSeconds;
|
||||||
|
|
||||||
|
if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining)
|
||||||
|
{
|
||||||
|
remaining = refreshed;
|
||||||
|
retryAt = DateTime.Now.AddSeconds(remaining);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static void SaveState()
|
private static void SaveState()
|
||||||
{
|
{
|
||||||
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
||||||
|
|||||||
@@ -736,19 +736,7 @@ namespace URLNotesGrabberCORE
|
|||||||
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
|
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
|
||||||
ApiKeyPool.MarkRateLimited(key, retry);
|
ApiKeyPool.MarkRateLimited(key, retry);
|
||||||
|
|
||||||
if (ApiKeyPool.IsAllRateLimited(out int minRetry))
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
||||||
{
|
|
||||||
Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before retry");
|
|
||||||
int remaining = minRetry;
|
|
||||||
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
|
|
||||||
while (remaining > 0)
|
|
||||||
{
|
|
||||||
Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
|
|
||||||
int sleepSeconds = Math.Min(60, remaining);
|
|
||||||
Thread.Sleep(sleepSeconds * 1000);
|
|
||||||
remaining -= sleepSeconds;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -969,19 +957,7 @@ namespace URLNotesGrabberCORE
|
|||||||
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
|
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
|
||||||
ApiKeyPool.MarkRateLimited(key, retry);
|
ApiKeyPool.MarkRateLimited(key, retry);
|
||||||
|
|
||||||
if (ApiKeyPool.IsAllRateLimited(out int minRetry))
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
||||||
{
|
|
||||||
Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before returning");
|
|
||||||
int remaining = minRetry;
|
|
||||||
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
|
|
||||||
while (remaining > 0)
|
|
||||||
{
|
|
||||||
Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
|
|
||||||
int sleepSeconds = Math.Min(60, remaining);
|
|
||||||
Thread.Sleep(sleepSeconds * 1000);
|
|
||||||
remaining -= sleepSeconds;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return response.statusCode;
|
return response.statusCode;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1093,6 +1069,8 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
while (posts.Count > 0)
|
while (posts.Count > 0)
|
||||||
{
|
{
|
||||||
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
||||||
|
|
||||||
var post = posts[0]; // Process the first post in the list
|
var post = posts[0]; // Process the first post in the list
|
||||||
string status;
|
string status;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user