jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make -collect 0 a resumable, single-pass full re-check

Browse Source 
Mode 0 (full re-check) previously reset its cutoff to now on every
launch, so an interrupted run restarted from scratch, and a post that
kept returning a non-Success/non-NotFound status could loop forever.

- Add single-row CollectRunState table + accessors (EnsureCollectRunStateTableExists,
  GetCollectRunState, BeginCollectRun, CompleteCollectRun) mirroring the
  ApiKeyPoolMeta pattern, to persist a frozen run cutoff and completion flag.
- -collect 0 with no explicit date is now a managed run: resume against the
  stored cutoff if a run is in progress, else start a new run; mark complete
  when the pass finishes so the next launch starts fresh. Explicit-date and
  mode 1 behavior unchanged.
- CollectNotes makes a single attempt pass via an in-process attempted set;
  FAILURE/UNKNOWN are logged once, TooManyRequests/no-lease aborts without
  completing so a later launch resumes.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jim
2026-06-03 15:48:26 -05:00
co-authored by Claude Opus 4.8
parent b576a9cdf3
commit 18f172fe96
2 changed files with 123 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+63
View File
@@ -1342,6 +1342,69 @@ namespace URLNotesGrabberCORE
}
}
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
// Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the
// same cutoff and lets a completed run stop instead of restarting on the next launch.
public static void EnsureCollectRunStateTableExists(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" +
"Id INTEGER PRIMARY KEY CHECK (Id = 1), " +
"RunCutoff INTEGER DEFAULT 0, " +
"RunComplete INTEGER DEFAULT 1, " +
"RunStarted TEXT, " +
"RunCompletedAt TEXT)";
using SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
// Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet.
public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection);
using SQLiteDataReader reader = command.ExecuteReader();
if (reader.Read())
{
long cutoff = Convert.ToInt64(reader.GetValue(0));
bool complete = Convert.ToInt64(reader.GetValue(1)) != 0;
return (cutoff, complete);
}
return null;
}
// Start (or restart) a managed run: freeze the cutoff and mark the run in progress.
public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " +
"VALUES (1, @cutoff, 0, @started, NULL) " +
"ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL";
using SQLiteCommand command = new SQLiteCommand(sql, connection);
command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds);
command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
// Mark the active managed run complete so the next launch starts fresh instead of resuming.
public static void CompleteCollectRun(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1";
using SQLiteCommand command = new SQLiteCommand(sql, connection);
command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
URLNotesGrabberCORE/Program.cs
+60 -5
View File
@@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -231,6 +231,7 @@ namespace URLNotesGrabberCORE
case "-collect": //collect notes from all posts
bool withoutNotesOnly = true;
DateTime? beforeDate = DateTime.Now;
bool explicitDateSupplied = false;
if (args.Length < 2)
{
@@ -261,6 +262,7 @@ namespace URLNotesGrabberCORE
if (DateTime.TryParse(args[2], out DateTime parsedDate))
{
beforeDate = parsedDate;
explicitDateSupplied = true;
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
}
else
@@ -270,7 +272,29 @@ namespace URLNotesGrabberCORE
}
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
// Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
bool managedCollectRun = false;
if (!withoutNotesOnly && !explicitDateSupplied)
{
DataAccess.EnsureCollectRunStateTableExists();
var runState = DataAccess.GetCollectRunState();
if (runState != null && !runState.Value.complete)
{
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
}
else
{
beforeDate = DateTime.Now;
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
}
managedCollectRun = true;
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
break;
case "-blogsR": //collect notes from all posts
@@ -1188,10 +1212,15 @@ if (shouldInsert)
return "UNKNOWN";
}
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
{
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{
PermitLimit = 300,
@@ -1208,9 +1237,17 @@ if (shouldInsert)
{
while (posts.Count > 0)
{
// First post not yet attempted this process. If all remaining have been attempted, the pass
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
if (post == null)
{
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
break;
}
ApiKeyPool.SleepUntilAnyAvailable(30);
var post = posts[0]; // Process the first post in the list
string status;
using RateLimitLease lease = limiter.AttemptAcquire(1);
@@ -1222,20 +1259,31 @@ if (shouldInsert)
else
{
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return;
return; // throttle: abort without completing the run so a later launch resumes
}
if (status == "Success")
{
attempted.Add((post.Item1, post.Item2));
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
}
else if (status == "NotFound")
{
attempted.Add((post.Item1, post.Item2));
Console.WriteLine("GrabNotes Result: NotFound");
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
}
else if (status == "TooManyRequests")
{
// Throttle, not a real per-post failure: don't consume this post's single attempt.
// Abort the pass without completing so a later launch resumes against the same cutoff.
Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
return;
}
else
{
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
attempted.Add((post.Item1, post.Item2));
Console.WriteLine("GrabNotes Result: " + status);
}
@@ -1243,6 +1291,13 @@ if (shouldInsert)
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
}
}
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
if (managedRun)
{
DataAccess.CompleteCollectRun();
Console.WriteLine("Full re-check run complete.");
}
}
catch (Exception ex)
{