From 5ae1d4feb878d45450a6069f4f3652decb609459 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2026 08:37:47 -0500 Subject: [PATCH 1/2] fix: skip API calls when all keys are rate-limited in -collect mode Pre-flight check on the CollectNotes loop now sleeps until at least one key recovers instead of issuing a wasted 429-bound request per iteration. Extracts the countdown into ApiKeyPool.SleepUntilAnyAvailable (30s refresh) and reuses it in CollectLikes and GrabNotes. Co-Authored-By: Claude Opus 4.7 --- URLNotesGrabberCORE/DataAccess.cs | 22 ++++++++++++++++++++++ URLNotesGrabberCORE/Program.cs | 30 ++++-------------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 2616a7c..95338f4 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -1948,6 +1948,28 @@ namespace URLNotesGrabberCORE return all; } + public static void SleepUntilAnyAvailable(int refreshSeconds = 30) + { + if (refreshSeconds <= 0) refreshSeconds = 30; + if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return; + + DateTime retryAt = DateTime.Now.AddSeconds(minRetry); + int remaining = minRetry; + while (remaining > 0) + { + Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T")); + int sleepSeconds = Math.Min(refreshSeconds, remaining); + Thread.Sleep(sleepSeconds * 1000); + remaining -= sleepSeconds; + + if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining) + { + remaining = refreshed; + retryAt = DateTime.Now.AddSeconds(remaining); + } + } + } + private static void SaveState() { using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index ddd70dd..7e593a1 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -736,19 +736,7 @@ namespace URLNotesGrabberCORE int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60; ApiKeyPool.MarkRateLimited(key, retry); - if (ApiKeyPool.IsAllRateLimited(out int minRetry)) - { - Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before retry"); - int remaining = minRetry; - DateTime retryAt = DateTime.Now.AddSeconds(minRetry); - while (remaining > 0) - { - Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString()); - int sleepSeconds = Math.Min(60, remaining); - Thread.Sleep(sleepSeconds * 1000); - remaining -= sleepSeconds; - } - } + ApiKeyPool.SleepUntilAnyAvailable(30); continue; } @@ -969,19 +957,7 @@ namespace URLNotesGrabberCORE int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60; ApiKeyPool.MarkRateLimited(key, retry); - if (ApiKeyPool.IsAllRateLimited(out int minRetry)) - { - Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before returning"); - int remaining = minRetry; - DateTime retryAt = DateTime.Now.AddSeconds(minRetry); - while (remaining > 0) - { - Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString()); - int sleepSeconds = Math.Min(60, remaining); - Thread.Sleep(sleepSeconds * 1000); - remaining -= sleepSeconds; - } - } + ApiKeyPool.SleepUntilAnyAvailable(30); return response.statusCode; } @@ -1093,6 +1069,8 @@ namespace URLNotesGrabberCORE { while (posts.Count > 0) { + ApiKeyPool.SleepUntilAnyAvailable(30); + var post = posts[0]; // Process the first post in the list string status; From 47497d02bf7d46c151d4756a7ef16f368b79ca02 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2026 09:03:01 -0500 Subject: [PATCH 2/2] tweak GetPosts beforeDate filter and broaden GetBlogsForLikes - GetPosts: match NotesGatheredDateTime = 0 instead of IS NULL for the beforeDate cutoff. - GetBlogsForLikes: drop the EXISTS-Posts predicate so blogs with notes but no posts rows are still eligible for likes collection. Co-Authored-By: Claude Opus 4.7 --- URLNotesGrabberCORE/DataAccess.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 95338f4..7ab83a3 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -665,7 +665,7 @@ namespace URLNotesGrabberCORE if (beforeDate.HasValue) { long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds(); - sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine; + sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime= 0) " + Environment.NewLine; } sql += "GROUP BY " + Environment.NewLine + @@ -972,7 +972,7 @@ namespace URLNotesGrabberCORE if (!string.IsNullOrEmpty(specificBlog)) sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog"; else - sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName AND EXISTS (SELECT 1 FROM Posts P WHERE P.BlogName = B.BlogName) GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);"; + sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) {