Make -collect 0 a resumable, single-pass full re-check

Mode 0 (full re-check) previously reset its cutoff to now on every
launch, so an interrupted run restarted from scratch, and a post that
kept returning a non-Success/non-NotFound status could loop forever.

- Add single-row CollectRunState table + accessors (EnsureCollectRunStateTableExists,
  GetCollectRunState, BeginCollectRun, CompleteCollectRun) mirroring the
  ApiKeyPoolMeta pattern, to persist a frozen run cutoff and completion flag.
- -collect 0 with no explicit date is now a managed run: resume against the
  stored cutoff if a run is in progress, else start a new run; mark complete
  when the pass finishes so the next launch starts fresh. Explicit-date and
  mode 1 behavior unchanged.
- CollectNotes makes a single attempt pass via an in-process attempted set;
  FAILURE/UNKNOWN are logged once, TooManyRequests/no-lease aborts without
  completing so a later launch resumes.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

URLNotesGrabberCORE/DataAccess.cs

@@ -1342,6 +1342,69 @@ namespace URLNotesGrabberCORE
             }
         }
 
+        // ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
+        // Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the
+        // same cutoff and lets a completed run stop instead of restarting on the next launch.
+
+        public static void EnsureCollectRunStateTableExists(string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+            using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" +
+                         "Id INTEGER PRIMARY KEY CHECK (Id = 1), " +
+                         "RunCutoff INTEGER DEFAULT 0, " +
+                         "RunComplete INTEGER DEFAULT 1, " +
+                         "RunStarted TEXT, " +
+                         "RunCompletedAt TEXT)";
+            using SQLiteCommand command = new SQLiteCommand(sql, connection);
+            command.ExecuteNonQuery();
+        }
+
+        // Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet.
+        public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+            using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection);
+            using SQLiteDataReader reader = command.ExecuteReader();
+            if (reader.Read())
+            {
+                long cutoff = Convert.ToInt64(reader.GetValue(0));
+                bool complete = Convert.ToInt64(reader.GetValue(1)) != 0;
+                return (cutoff, complete);
+            }
+            return null;
+        }
+
+        // Start (or restart) a managed run: freeze the cutoff and mark the run in progress.
+        public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+            using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " +
+                         "VALUES (1, @cutoff, 0, @started, NULL) " +
+                         "ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL";
+            using SQLiteCommand command = new SQLiteCommand(sql, connection);
+            command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds);
+            command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+            command.ExecuteNonQuery();
+        }
+
+        // Mark the active managed run complete so the next launch starts fresh instead of resuming.
+        public static void CompleteCollectRun(string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+            using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1";
+            using SQLiteCommand command = new SQLiteCommand(sql, connection);
+            command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+            command.ExecuteNonQuery();
+        }
+
         public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();

URLNotesGrabberCORE/Program.cs

@@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE
 
                         Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
 
-                        Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
+                        Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
 
                         Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
 
@@ -231,6 +231,7 @@ namespace URLNotesGrabberCORE
                     case "-collect": //collect notes from all posts
                         bool withoutNotesOnly = true;
                         DateTime? beforeDate = DateTime.Now;
+                        bool explicitDateSupplied = false;
 
                         if (args.Length < 2)
                         {
@@ -261,6 +262,7 @@ namespace URLNotesGrabberCORE
                             if (DateTime.TryParse(args[2], out DateTime parsedDate))
                             {
                                 beforeDate = parsedDate;
+                                explicitDateSupplied = true;
                                 Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
                             }
                             else
@@ -270,7 +272,29 @@ namespace URLNotesGrabberCORE
                             }
                         }
 
-                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
+                        // Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
+                        // persist it so an interrupted run resumes against the same cutoff and a completed run stops
+                        // instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
+                        bool managedCollectRun = false;
+                        if (!withoutNotesOnly && !explicitDateSupplied)
+                        {
+                            DataAccess.EnsureCollectRunStateTableExists();
+                            var runState = DataAccess.GetCollectRunState();
+                            if (runState != null && !runState.Value.complete)
+                            {
+                                beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
+                                Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
+                            }
+                            else
+                            {
+                                beforeDate = DateTime.Now;
+                                DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
+                                Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
+                            }
+                            managedCollectRun = true;
+                        }
+
+                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
                         break;
 
                     case "-blogsR": //collect notes from all posts
@@ -1188,10 +1212,15 @@ if (shouldInsert)
             return "UNKNOWN";
         }
 
-        static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
+        static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
         {
             List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
 
+            // Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
+            // attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
+            // post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
+            HashSet<(string, long)> attempted = new HashSet<(string, long)>();
+
             RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
             {
                 PermitLimit = 300,
@@ -1208,9 +1237,17 @@ if (shouldInsert)
                 {
                     while (posts.Count > 0)
                     {
+                        // First post not yet attempted this process. If all remaining have been attempted, the pass
+                        // is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
+                        var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
+                        if (post == null)
+                        {
+                            Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
+                            break;
+                        }
+
                         ApiKeyPool.SleepUntilAnyAvailable(30);
 
-                        var post = posts[0]; // Process the first post in the list
                         string status;
 
                         using RateLimitLease lease = limiter.AttemptAcquire(1);
@@ -1222,20 +1259,31 @@ if (shouldInsert)
                         else
                         {
                             Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
-                            return;
+                            return; // throttle: abort without completing the run so a later launch resumes
                         }
 
                         if (status == "Success")
                         {
+                            attempted.Add((post.Item1, post.Item2));
                             DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
                         }
                         else if (status == "NotFound")
                         {
+                            attempted.Add((post.Item1, post.Item2));
                             Console.WriteLine("GrabNotes Result: NotFound");
                             DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                         }
+                        else if (status == "TooManyRequests")
+                        {
+                            // Throttle, not a real per-post failure: don't consume this post's single attempt.
+                            // Abort the pass without completing so a later launch resumes against the same cutoff.
+                            Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
+                            return;
+                        }
                         else
                         {
+                            // FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
+                            attempted.Add((post.Item1, post.Item2));
                             Console.WriteLine("GrabNotes Result: " + status);
                         }
 
@@ -1243,6 +1291,13 @@ if (shouldInsert)
                         posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
                     }
                 }
+
+                // Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
+                if (managedRun)
+                {
+                    DataAccess.CompleteCollectRun();
+                    Console.WriteLine("Full re-check run complete.");
+                }
             }
             catch (Exception ex)
             {