diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index bded7c1..99e9490 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -1342,6 +1342,69 @@ namespace URLNotesGrabberCORE } } + // ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run ----- + // Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the + // same cutoff and lets a completed run stop instead of restarting on the next launch. + + public static void EnsureCollectRunStateTableExists(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" + + "Id INTEGER PRIMARY KEY CHECK (Id = 1), " + + "RunCutoff INTEGER DEFAULT 0, " + + "RunComplete INTEGER DEFAULT 1, " + + "RunStarted TEXT, " + + "RunCompletedAt TEXT)"; + using SQLiteCommand command = new SQLiteCommand(sql, connection); + command.ExecuteNonQuery(); + } + + // Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet. + public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection); + using SQLiteDataReader reader = command.ExecuteReader(); + if (reader.Read()) + { + long cutoff = Convert.ToInt64(reader.GetValue(0)); + bool complete = Convert.ToInt64(reader.GetValue(1)) != 0; + return (cutoff, complete); + } + return null; + } + + // Start (or restart) a managed run: freeze the cutoff and mark the run in progress. + public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " + + "VALUES (1, @cutoff, 0, @started, NULL) " + + "ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL"; + using SQLiteCommand command = new SQLiteCommand(sql, connection); + command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds); + command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.ExecuteNonQuery(); + } + + // Mark the active managed run complete so the next launch starts fresh instead of resuming. + public static void CompleteCollectRun(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1"; + using SQLiteCommand command = new SQLiteCommand(sql, connection); + command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.ExecuteNonQuery(); + } + public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 11fabfc..018bc5f 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file"); - Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime"); + Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking)."); Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file "); @@ -231,6 +231,7 @@ namespace URLNotesGrabberCORE case "-collect": //collect notes from all posts bool withoutNotesOnly = true; DateTime? beforeDate = DateTime.Now; + bool explicitDateSupplied = false; if (args.Length < 2) { @@ -261,6 +262,7 @@ namespace URLNotesGrabberCORE if (DateTime.TryParse(args[2], out DateTime parsedDate)) { beforeDate = parsedDate; + explicitDateSupplied = true; Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}"); } else @@ -270,7 +272,29 @@ namespace URLNotesGrabberCORE } } - CollectNotes(settings.GetValue("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult(); + // Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and + // persist it so an interrupted run resumes against the same cutoff and a completed run stops + // instead of restarting. Mode 1 and explicit-date runs keep their existing behavior. + bool managedCollectRun = false; + if (!withoutNotesOnly && !explicitDateSupplied) + { + DataAccess.EnsureCollectRunStateTableExists(); + var runState = DataAccess.GetCollectRunState(); + if (runState != null && !runState.Value.complete) + { + beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime; + Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})"); + } + else + { + beforeDate = DateTime.Now; + DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds()); + Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})"); + } + managedCollectRun = true; + } + + CollectNotes(settings.GetValue("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult(); break; case "-blogsR": //collect notes from all posts @@ -1188,10 +1212,15 @@ if (shouldInsert) return "UNKNOWN"; } - static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null) + static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false) { List> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate); + // Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single + // attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a + // post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway. + HashSet<(string, long)> attempted = new HashSet<(string, long)>(); + RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { PermitLimit = 300, @@ -1208,9 +1237,17 @@ if (shouldInsert) { while (posts.Count > 0) { + // First post not yet attempted this process. If all remaining have been attempted, the pass + // is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever. + var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2))); + if (post == null) + { + Console.WriteLine("All remaining posts have been attempted this run; ending pass."); + break; + } + ApiKeyPool.SleepUntilAnyAvailable(30); - var post = posts[0]; // Process the first post in the list string status; using RateLimitLease lease = limiter.AttemptAcquire(1); @@ -1222,20 +1259,31 @@ if (shouldInsert) else { Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); - return; + return; // throttle: abort without completing the run so a later launch resumes } if (status == "Success") { + attempted.Add((post.Item1, post.Item2)); DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); } else if (status == "NotFound") { + attempted.Add((post.Item1, post.Item2)); Console.WriteLine("GrabNotes Result: NotFound"); DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); } + else if (status == "TooManyRequests") + { + // Throttle, not a real per-post failure: don't consume this post's single attempt. + // Abort the pass without completing so a later launch resumes against the same cutoff. + Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume."); + return; + } else { + // FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever. + attempted.Add((post.Item1, post.Item2)); Console.WriteLine("GrabNotes Result: " + status); } @@ -1243,6 +1291,13 @@ if (shouldInsert) posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate); } } + + // Reached only when the pass finished naturally (worklist drained or all stragglers attempted). + if (managedRun) + { + DataAccess.CompleteCollectRun(); + Console.WriteLine("Full re-check run complete."); + } } catch (Exception ex) {