jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(collect): add --fromDate lower bound for zomb-eh re-queue branch

Browse Source 
The zomb-eh periodic re-queue branch in GetPosts had no lower bound on
the post's original PostDate -- it re-queued every already-collected
zomb-eh post past the 3-day cooldown, regardless of age.

Add --fromDate <datetime> to bound that branch by PostDate >= the given
date. It stacks with the existing cooldown clause rather than replacing
it, so it applies the same way whether or not --force also drops the
cooldown.
This commit is contained in:
jim
2026-08-24 14:42:43 -05:00
parent 58d7b1d05e
commit 7bf270e47d
3 changed files with 47 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+27 -5
View File
@@ -44,6 +44,7 @@ namespace URLNotesGrabberCORE
bool apiExplicitlySet = false;
string startFromBlogName = string.Empty;
bool forceIgnoreCooldown = false;
DateTime? fromDate = null;
List<string> filteredArgs = new List<string>();
for (int i = 0; i < args.Length; i++)
{
@@ -60,6 +61,20 @@ namespace URLNotesGrabberCORE
continue;
}
if (string.Equals(args[i], "--fromDate", StringComparison.OrdinalIgnoreCase))
{
if (i + 1 < args.Length && DateTime.TryParse(args[i + 1], out DateTime parsedFromDate))
{
fromDate = parsedFromDate;
i++;
}
else
{
Console.WriteLine("--Missing or unparseable date after --fromDate. Ignoring.--");
}
continue;
}
if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
{
apiSectionName = "TumblrApi3";
@@ -327,7 +342,12 @@ namespace URLNotesGrabberCORE
? "--force: ignoring the periodic re-collect cooldown; already-collected posts in scope are re-queued now"
: "--force: no effect in mode 0 - a full re-check already re-collects every post");
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown).GetAwaiter().GetResult();
if (fromDate.HasValue)
Console.WriteLine(withoutNotesOnly
? $"--fromDate: only re-queuing already-collected posts originally posted on/after {fromDate.Value} (applies with or without --force)"
: "--fromDate: no effect in mode 0 - it only bounds the periodic re-queue branch");
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown, fromDate).GetAwaiter().GetResult();
break;
case "--blogsR": //collect notes from all posts
@@ -459,7 +479,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only).");
Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only). Add --fromDate <datetime> to only re-queue already-collected posts originally posted on/after that date (mode 1 only; applies with or without --force).");
Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -473,6 +493,8 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
Console.WriteLine("--fromDate <datetime>\t With --collect 1, only re-queue already-collected posts originally posted on/after <datetime>. Independent of --force - applies whether or not the cooldown is also bypassed.");
Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json");
@@ -1350,9 +1372,9 @@ if (shouldInsert)
// blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
const int MaxConsecutiveTransient = 10;
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false)
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null)
{
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown);
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate);
if (posts.Count == 0 && !string.IsNullOrWhiteSpace(blogName))
{
@@ -1458,7 +1480,7 @@ if (shouldInsert)
}
// Re-fetch the updated list after processing the current post
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown);
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate);
}
}