jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'claude/goofy-antonelli-73ea26'

Browse Source
This commit is contained in:
jim
2026-05-11 09:03:12 -05:00
parent 2034b43e83 47497d02bf
commit a5c118ce3b
2 changed files with 28 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+24 -2
View File
@@ -665,7 +665,7 @@ namespace URLNotesGrabberCORE
if (beforeDate.HasValue)
{
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime= 0) " + Environment.NewLine;
}
sql += "GROUP BY " + Environment.NewLine +
@@ -972,7 +972,7 @@ namespace URLNotesGrabberCORE
if (!string.IsNullOrEmpty(specificBlog))
sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
else
sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName AND EXISTS (SELECT 1 FROM Posts P WHERE P.BlogName = B.BlogName) GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1948,6 +1948,28 @@ namespace URLNotesGrabberCORE
return all;
}
public static void SleepUntilAnyAvailable(int refreshSeconds = 30)
{
if (refreshSeconds <= 0) refreshSeconds = 30;
if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return;
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
int remaining = minRetry;
while (remaining > 0)
{
Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T"));
int sleepSeconds = Math.Min(refreshSeconds, remaining);
Thread.Sleep(sleepSeconds * 1000);
remaining -= sleepSeconds;
if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining)
{
remaining = refreshed;
retryAt = DateTime.Now.AddSeconds(remaining);
}
}
}
private static void SaveState()
{
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
URLNotesGrabberCORE/Program.cs
+4 -26
View File
@@ -736,19 +736,7 @@ namespace URLNotesGrabberCORE
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
if (ApiKeyPool.IsAllRateLimited(out int minRetry))
{
Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before retry");
int remaining = minRetry;
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
while (remaining > 0)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
int sleepSeconds = Math.Min(60, remaining);
Thread.Sleep(sleepSeconds * 1000);
remaining -= sleepSeconds;
}
}
ApiKeyPool.SleepUntilAnyAvailable(30);
continue;
}
@@ -969,19 +957,7 @@ namespace URLNotesGrabberCORE
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry);
if (ApiKeyPool.IsAllRateLimited(out int minRetry))
{
Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before returning");
int remaining = minRetry;
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
while (remaining > 0)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
int sleepSeconds = Math.Min(60, remaining);
Thread.Sleep(sleepSeconds * 1000);
remaining -= sleepSeconds;
}
}
ApiKeyPool.SleepUntilAnyAvailable(30);
return response.statusCode;
}
@@ -1093,6 +1069,8 @@ namespace URLNotesGrabberCORE
{
while (posts.Count > 0)
{
ApiKeyPool.SleepUntilAnyAvailable(30);
var post = posts[0]; // Process the first post in the list
string status;