From 7bf270e47d1a197448149dccc6d8c527921ef680 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 24 Aug 2026 14:42:43 -0500 Subject: [PATCH] feat(collect): add --fromDate lower bound for zomb-eh re-queue branch The zomb-eh periodic re-queue branch in GetPosts had no lower bound on the post's original PostDate -- it re-queued every already-collected zomb-eh post past the 3-day cooldown, regardless of age. Add --fromDate to bound that branch by PostDate >= the given date. It stacks with the existing cooldown clause rather than replacing it, so it applies the same way whether or not --force also drops the cooldown. --- .github/copilot-instructions.md | 2 +- URLNotesGrabberCORE/DataAccess.cs | 22 ++++++++++++++++++--- URLNotesGrabberCORE/Program.cs | 32 ++++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 09e49db..80df20f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -31,7 +31,7 @@ dotnet run -- --test [blogname] [postID] # Test API for specific post - `--test [blogname] [postID]`: Test API note collection - `--posts`: Export post blogs to file - `--blogs`: Export blog list to file -- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`. Add `--force` to ignore the periodic re-collect cooldown so already-collected posts are re-queued immediately (mode 1 only) +- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`. Add `--force` to ignore the periodic re-collect cooldown so already-collected posts are re-queued immediately (mode 1 only). Add `--fromDate ` to only re-queue already-collected posts whose original PostDate is on/after that date (mode 1 only; applies with or without `--force`) - `--blogsR`: Export reply blogs to file - `--blogsO [start] [stop]`: Export blogs within range diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 474c0cb..c67456d 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -855,13 +855,14 @@ namespace URLNotesGrabberCORE #region Gets /// - /// + /// /// /// /// Drops the age gate on the periodic re-queue branch (--force). + /// Lower bound on the *original post's* PostDate for the periodic re-queue branch (--fromDate). Independent of ignoreRefreshCooldown -- applies whether or not --force is also given. /// /// blogName, postID, lastNoteTimestamp, notesGatheredTimestamp - public static List> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, bool ignoreRefreshCooldown = false, string? DBPath = null) + public static List> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -903,6 +904,15 @@ namespace URLNotesGrabberCORE ? string.Empty : " AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine; + // --fromDate bounds the *original post's* PostDate, not the re-collect cooldown -- + // it stacks with refreshCooldownClause instead of replacing it, so it applies the + // same way whether or not --force also dropped the cooldown. A NULL PostDate never + // satisfies ">=" and is excluded, same as an unfiltered run would still include it + // (there's nothing to compare here, so this only narrows, never widens, the result). + string fromDateClause = fromDate.HasValue + ? " AND PostDate >= @fromDate" + Environment.NewLine + : string.Empty; + string refreshBranch = "" + Environment.NewLine + " UNION " + Environment.NewLine + @@ -917,7 +927,8 @@ namespace URLNotesGrabberCORE " FROM PostsWithCount" + Environment.NewLine + " WHERE BlogName = 'zomb-eh'" + Environment.NewLine + " AND NotFound = 0" + Environment.NewLine + - refreshCooldownClause; + refreshCooldownClause + + fromDateClause; sql = "WITH PostsWithCount AS" + Environment.NewLine + "(" + Environment.NewLine + @@ -1001,6 +1012,11 @@ namespace URLNotesGrabberCORE if (filterByBlog) command.Parameters.AddWithValue("@blogName", blogName); + // Only ever referenced by the zomb-eh refresh branch, which only exists when + // withoutNotesOnly is true -- harmless to bind unconditionally otherwise. + if (fromDate.HasValue) + command.Parameters.AddWithValue("@fromDate", fromDate.Value.ToString("yyyy-MM-dd HH:mm:ss")); + using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 27149f8..616f6bd 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -44,6 +44,7 @@ namespace URLNotesGrabberCORE bool apiExplicitlySet = false; string startFromBlogName = string.Empty; bool forceIgnoreCooldown = false; + DateTime? fromDate = null; List filteredArgs = new List(); for (int i = 0; i < args.Length; i++) { @@ -60,6 +61,20 @@ namespace URLNotesGrabberCORE continue; } + if (string.Equals(args[i], "--fromDate", StringComparison.OrdinalIgnoreCase)) + { + if (i + 1 < args.Length && DateTime.TryParse(args[i + 1], out DateTime parsedFromDate)) + { + fromDate = parsedFromDate; + i++; + } + else + { + Console.WriteLine("--Missing or unparseable date after --fromDate. Ignoring.--"); + } + continue; + } + if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase)) { apiSectionName = "TumblrApi3"; @@ -327,7 +342,12 @@ namespace URLNotesGrabberCORE ? "--force: ignoring the periodic re-collect cooldown; already-collected posts in scope are re-queued now" : "--force: no effect in mode 0 - a full re-check already re-collects every post"); - exitCode = CollectNotes(settings.GetValue("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown).GetAwaiter().GetResult(); + if (fromDate.HasValue) + Console.WriteLine(withoutNotesOnly + ? $"--fromDate: only re-queuing already-collected posts originally posted on/after {fromDate.Value} (applies with or without --force)" + : "--fromDate: no effect in mode 0 - it only bounds the periodic re-queue branch"); + + exitCode = CollectNotes(settings.GetValue("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown, fromDate).GetAwaiter().GetResult(); break; case "--blogsR": //collect notes from all posts @@ -459,7 +479,7 @@ namespace URLNotesGrabberCORE Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file"); - Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only)."); + Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only). Add --fromDate to only re-queue already-collected posts originally posted on/after that date (mode 1 only; applies with or without --force)."); Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file "); @@ -473,6 +493,8 @@ namespace URLNotesGrabberCORE Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown"); + Console.WriteLine("--fromDate \t With --collect 1, only re-queue already-collected posts originally posted on/after . Independent of --force - applies whether or not the cooldown is also bypassed."); + Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file"); Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json"); @@ -1350,9 +1372,9 @@ if (shouldInsert) // blipping on one post. Past this, skipping post-by-post would just hammer a closed door. const int MaxConsecutiveTransient = 10; - static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false) + static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null) { - List> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown); + List> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate); if (posts.Count == 0 && !string.IsNullOrWhiteSpace(blogName)) { @@ -1458,7 +1480,7 @@ if (shouldInsert) } // Re-fetch the updated list after processing the current post - posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown); + posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate); } }