Merge branch 'claude/goofy-antonelli-73ea26'

URLNotesGrabberCORE/DataAccess.cs

@@ -665,7 +665,7 @@ namespace URLNotesGrabberCORE
                     if (beforeDate.HasValue)
                     {
                         long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
-                        sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
+                        sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime= 0) " + Environment.NewLine;
                     }
 
                     sql += "GROUP BY " + Environment.NewLine +
@@ -972,7 +972,7 @@ namespace URLNotesGrabberCORE
                 if (!string.IsNullOrEmpty(specificBlog))
                     sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
                 else
-                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName AND EXISTS (SELECT 1 FROM Posts P WHERE P.BlogName = B.BlogName) GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
+                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1948,6 +1948,28 @@ namespace URLNotesGrabberCORE
             return all;
         }
 
+        public static void SleepUntilAnyAvailable(int refreshSeconds = 30)
+        {
+            if (refreshSeconds <= 0) refreshSeconds = 30;
+            if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return;
+
+            DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
+            int remaining = minRetry;
+            while (remaining > 0)
+            {
+                Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T"));
+                int sleepSeconds = Math.Min(refreshSeconds, remaining);
+                Thread.Sleep(sleepSeconds * 1000);
+                remaining -= sleepSeconds;
+
+                if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining)
+                {
+                    remaining = refreshed;
+                    retryAt = DateTime.Now.AddSeconds(remaining);
+                }
+            }
+        }
+
         private static void SaveState()
         {
             using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);

URLNotesGrabberCORE/Program.cs

@@ -736,19 +736,7 @@ namespace URLNotesGrabberCORE
                             int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                             ApiKeyPool.MarkRateLimited(key, retry);
 
-                            if (ApiKeyPool.IsAllRateLimited(out int minRetry))
+                            ApiKeyPool.SleepUntilAnyAvailable(30);
-                            {
-                                Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before retry");
-                                int remaining = minRetry;
-                                DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
-                                while (remaining > 0)
-                                {
-                                    Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
-                                    int sleepSeconds = Math.Min(60, remaining);
-                                    Thread.Sleep(sleepSeconds * 1000);
-                                    remaining -= sleepSeconds;
-                                }
-                            }
                             continue;
                         }
 
@@ -969,19 +957,7 @@ namespace URLNotesGrabberCORE
                     int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                     ApiKeyPool.MarkRateLimited(key, retry);
 
-                    if (ApiKeyPool.IsAllRateLimited(out int minRetry))
+                    ApiKeyPool.SleepUntilAnyAvailable(30);
-                    {
-                        Console.WriteLine($"[Pool] All keys rate-limited, waiting {minRetry}s before returning");
-                        int remaining = minRetry;
-                        DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
-                        while (remaining > 0)
-                        {
-                            Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryAt.ToShortTimeString());
-                            int sleepSeconds = Math.Min(60, remaining);
-                            Thread.Sleep(sleepSeconds * 1000);
-                            remaining -= sleepSeconds;
-                        }
-                    }
                     return response.statusCode;
                 }
 
@@ -1093,6 +1069,8 @@ namespace URLNotesGrabberCORE
                 {
                     while (posts.Count > 0)
                     {
+                        ApiKeyPool.SleepUntilAnyAvailable(30);
+
                         var post = posts[0]; // Process the first post in the list
                         string status;