jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(collect): add --toDate upper bound for zomb-eh re-queue branch

Browse Source 
Mirrors --fromDate: bounds the zomb-eh periodic re-queue branch by
PostDate <= the given date. Stacks alongside --fromDate and the
cooldown clause rather than replacing either, so --fromDate/--toDate
and --force can all be combined, and each applies with or without
--force.

Named --toDate rather than --end to pair with --fromDate -- --start
already exists as an unrelated flag (resume folder traversal at a
blog name).
This commit is contained in:
jim
2026-08-25 12:51:32 -05:00
parent 7bf270e47d
commit ea2afc9d35
3 changed files with 41 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/copilot-instructions.md
+1 -1
View File
@@ -31,7 +31,7 @@ dotnet run -- --test [blogname] [postID] # Test API for specific post
- `--test [blogname] [postID]`: Test API note collection
- `--posts`: Export post blogs to file
- `--blogs`: Export blog list to file
- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`. Add `--force` to ignore the periodic re-collect cooldown so already-collected posts are re-queued immediately (mode 1 only). Add `--fromDate <datetime>` to only re-queue already-collected posts whose original PostDate is on/after that date (mode 1 only; applies with or without `--force`)
- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`. Add `--force` to ignore the periodic re-collect cooldown so already-collected posts are re-queued immediately (mode 1 only). Add `--fromDate <datetime>` / `--toDate <datetime>` to only re-queue already-collected posts whose original PostDate is on/after / on/before that date (mode 1 only; either or both may be given; applies with or without `--force`)
- `--blogsR`: Export reply blogs to file
- `--blogsO [start] [stop]`: Export blogs within range
URLNotesGrabberCORE/DataAccess.cs
+13 -2
View File
@@ -860,9 +860,10 @@ namespace URLNotesGrabberCORE
/// <param name="withoutNotesOnly"></param>
/// <param name="ignoreRefreshCooldown">Drops the age gate on the periodic re-queue branch (--force).</param>
/// <param name="fromDate">Lower bound on the *original post's* PostDate for the periodic re-queue branch (--fromDate). Independent of ignoreRefreshCooldown -- applies whether or not --force is also given.</param>
/// <param name="toDate">Upper bound on the *original post's* PostDate for the periodic re-queue branch (--toDate). Same independence from ignoreRefreshCooldown as fromDate.</param>
/// <param name="DBPath"></param>
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null, string? DBPath = null)
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null, DateTime? toDate = null, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -913,6 +914,13 @@ namespace URLNotesGrabberCORE
? " AND PostDate >= @fromDate" + Environment.NewLine
: string.Empty;
// --toDate is the same deal, mirrored: stacks alongside fromDateClause/
// refreshCooldownClause rather than replacing either, so --fromDate and --toDate
// can be given together (or alone) and both hold with or without --force.
string toDateClause = toDate.HasValue
? " AND PostDate <= @toDate" + Environment.NewLine
: string.Empty;
string refreshBranch =
"" + Environment.NewLine +
" UNION " + Environment.NewLine +
@@ -928,7 +936,8 @@ namespace URLNotesGrabberCORE
" WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
" AND NotFound = 0" + Environment.NewLine +
refreshCooldownClause +
fromDateClause;
fromDateClause +
toDateClause;
sql = "WITH PostsWithCount AS" + Environment.NewLine +
"(" + Environment.NewLine +
@@ -1016,6 +1025,8 @@ namespace URLNotesGrabberCORE
// withoutNotesOnly is true -- harmless to bind unconditionally otherwise.
if (fromDate.HasValue)
command.Parameters.AddWithValue("@fromDate", fromDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
if (toDate.HasValue)
command.Parameters.AddWithValue("@toDate", toDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
using (SQLiteDataReader reader = command.ExecuteReader())
{
URLNotesGrabberCORE/Program.cs
+27 -5
View File
@@ -45,6 +45,7 @@ namespace URLNotesGrabberCORE
string startFromBlogName = string.Empty;
bool forceIgnoreCooldown = false;
DateTime? fromDate = null;
DateTime? toDate = null;
List<string> filteredArgs = new List<string>();
for (int i = 0; i < args.Length; i++)
{
@@ -75,6 +76,20 @@ namespace URLNotesGrabberCORE
continue;
}
if (string.Equals(args[i], "--toDate", StringComparison.OrdinalIgnoreCase))
{
if (i + 1 < args.Length && DateTime.TryParse(args[i + 1], out DateTime parsedToDate))
{
toDate = parsedToDate;
i++;
}
else
{
Console.WriteLine("--Missing or unparseable date after --toDate. Ignoring.--");
}
continue;
}
if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
{
apiSectionName = "TumblrApi3";
@@ -347,7 +362,12 @@ namespace URLNotesGrabberCORE
? $"--fromDate: only re-queuing already-collected posts originally posted on/after {fromDate.Value} (applies with or without --force)"
: "--fromDate: no effect in mode 0 - it only bounds the periodic re-queue branch");
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown, fromDate).GetAwaiter().GetResult();
if (toDate.HasValue)
Console.WriteLine(withoutNotesOnly
? $"--toDate: only re-queuing already-collected posts originally posted on/before {toDate.Value} (applies with or without --force)"
: "--toDate: no effect in mode 0 - it only bounds the periodic re-queue branch");
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown, fromDate, toDate).GetAwaiter().GetResult();
break;
case "--blogsR": //collect notes from all posts
@@ -479,7 +499,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only). Add --fromDate <datetime> to only re-queue already-collected posts originally posted on/after that date (mode 1 only; applies with or without --force).");
Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only). Add --fromDate <datetime> / --toDate <datetime> to only re-queue already-collected posts originally posted on/after / on/before that date (mode 1 only; either or both may be given; applies with or without --force).");
Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -495,6 +515,8 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--fromDate <datetime>\t With --collect 1, only re-queue already-collected posts originally posted on/after <datetime>. Independent of --force - applies whether or not the cooldown is also bypassed.");
Console.WriteLine("--toDate <datetime>\t With --collect 1, only re-queue already-collected posts originally posted on/before <datetime>. Independent of --force; may be combined with --fromDate for a range.");
Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json");
@@ -1372,9 +1394,9 @@ if (shouldInsert)
// blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
const int MaxConsecutiveTransient = 10;
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null)
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null, DateTime? toDate = null)
{
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate);
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate, toDate);
if (posts.Count == 0 && !string.IsNullOrWhiteSpace(blogName))
{
@@ -1480,7 +1502,7 @@ if (shouldInsert)
}
// Re-fetch the updated list after processing the current post
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate);
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate, toDate);
}
}