jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make -collect 0 a resumable, single-pass full re-check

Browse Source 
Mode 0 (full re-check) previously reset its cutoff to now on every
launch, so an interrupted run restarted from scratch, and a post that
kept returning a non-Success/non-NotFound status could loop forever.

- Add single-row CollectRunState table + accessors (EnsureCollectRunStateTableExists,
  GetCollectRunState, BeginCollectRun, CompleteCollectRun) mirroring the
  ApiKeyPoolMeta pattern, to persist a frozen run cutoff and completion flag.
- -collect 0 with no explicit date is now a managed run: resume against the
  stored cutoff if a run is in progress, else start a new run; mark complete
  when the pass finishes so the next launch starts fresh. Explicit-date and
  mode 1 behavior unchanged.
- CollectNotes makes a single attempt pass via an in-process attempted set;
  FAILURE/UNKNOWN are logged once, TooManyRequests/no-lease aborts without
  completing so a later launch resumes.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jim
2026-06-03 15:48:26 -05:00
co-authored by Claude Opus 4.8
parent b576a9cdf3
commit 18f172fe96
2 changed files with 123 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+60 -5
View File
@@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -231,6 +231,7 @@ namespace URLNotesGrabberCORE
case "-collect": //collect notes from all posts
bool withoutNotesOnly = true;
DateTime? beforeDate = DateTime.Now;
bool explicitDateSupplied = false;
if (args.Length < 2)
{
@@ -261,6 +262,7 @@ namespace URLNotesGrabberCORE
if (DateTime.TryParse(args[2], out DateTime parsedDate))
{
beforeDate = parsedDate;
explicitDateSupplied = true;
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
}
else
@@ -270,7 +272,29 @@ namespace URLNotesGrabberCORE
}
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
// Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
bool managedCollectRun = false;
if (!withoutNotesOnly && !explicitDateSupplied)
{
DataAccess.EnsureCollectRunStateTableExists();
var runState = DataAccess.GetCollectRunState();
if (runState != null && !runState.Value.complete)
{
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
}
else
{
beforeDate = DateTime.Now;
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
}
managedCollectRun = true;
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
break;
case "-blogsR": //collect notes from all posts
@@ -1188,10 +1212,15 @@ if (shouldInsert)
return "UNKNOWN";
}
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
{
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{
PermitLimit = 300,
@@ -1208,9 +1237,17 @@ if (shouldInsert)
{
while (posts.Count > 0)
{
// First post not yet attempted this process. If all remaining have been attempted, the pass
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
if (post == null)
{
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
break;
}
ApiKeyPool.SleepUntilAnyAvailable(30);
var post = posts[0]; // Process the first post in the list
string status;
using RateLimitLease lease = limiter.AttemptAcquire(1);
@@ -1222,20 +1259,31 @@ if (shouldInsert)
else
{
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return;
return; // throttle: abort without completing the run so a later launch resumes
}
if (status == "Success")
{
attempted.Add((post.Item1, post.Item2));
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
}
else if (status == "NotFound")
{
attempted.Add((post.Item1, post.Item2));
Console.WriteLine("GrabNotes Result: NotFound");
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
}
else if (status == "TooManyRequests")
{
// Throttle, not a real per-post failure: don't consume this post's single attempt.
// Abort the pass without completing so a later launch resumes against the same cutoff.
Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
return;
}
else
{
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
attempted.Add((post.Item1, post.Item2));
Console.WriteLine("GrabNotes Result: " + status);
}
@@ -1243,6 +1291,13 @@ if (shouldInsert)
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
}
}
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
if (managedRun)
{
DataAccess.CompleteCollectRun();
Console.WriteLine("Full re-check run complete.");
}
}
catch (Exception ex)
{