jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'claude/collect-command-issue-09fcb8' into master

Browse Source
This commit is contained in:
jim
2026-08-23 04:34:50 -05:00
parent a28c5cc9ec f41957fd2f
commit 58d7b1d05e
3 changed files with 24 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/copilot-instructions.md
+1 -1
View File
@@ -31,7 +31,7 @@ dotnet run -- --test [blogname] [postID] # Test API for specific post
- `--test [blogname] [postID]`: Test API note collection - `--test [blogname] [postID]`: Test API note collection
- `--posts`: Export post blogs to file - `--posts`: Export post blogs to file
- `--blogs`: Export blog list to file - `--blogs`: Export blog list to file
- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh` - `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`. Add `--force` to ignore the periodic re-collect cooldown so already-collected posts are re-queued immediately (mode 1 only)
- `--blogsR`: Export reply blogs to file - `--blogsR`: Export reply blogs to file
- `--blogsO [start] [stop]`: Export blogs within range - `--blogsO [start] [stop]`: Export blogs within range
URLNotesGrabberCORE/DataAccess.cs
+10 -2
View File
@@ -858,9 +858,10 @@ namespace URLNotesGrabberCORE
/// ///
/// </summary> /// </summary>
/// <param name="withoutNotesOnly"></param> /// <param name="withoutNotesOnly"></param>
/// <param name="ignoreRefreshCooldown">Drops the age gate on the periodic re-queue branch (--force).</param>
/// <param name="DBPath"></param> /// <param name="DBPath"></param>
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns> /// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, string? DBPath = null) public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, bool ignoreRefreshCooldown = false, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -895,6 +896,13 @@ namespace URLNotesGrabberCORE
// scoped, so it contributes its rows when the filter names zomb-eh and nothing otherwise. // scoped, so it contributes its rows when the filter names zomb-eh and nothing otherwise.
// That keeps a filtered worklist a strict subset of the unfiltered one -- "--collect 1 X" // That keeps a filtered worklist a strict subset of the unfiltered one -- "--collect 1 X"
// returns exactly the rows "--collect 1" would have returned for X. // returns exactly the rows "--collect 1" would have returned for X.
//
// --force drops the age gate only. NotFound = 0 and the IsActive/blog scoping above still
// apply: the flag is "re-collect early", not "collect rows every other path excludes".
string refreshCooldownClause = ignoreRefreshCooldown
? string.Empty
: " AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine;
string refreshBranch = string refreshBranch =
"" + Environment.NewLine + "" + Environment.NewLine +
" UNION " + Environment.NewLine + " UNION " + Environment.NewLine +
@@ -909,7 +917,7 @@ namespace URLNotesGrabberCORE
" FROM PostsWithCount" + Environment.NewLine + " FROM PostsWithCount" + Environment.NewLine +
" WHERE BlogName = 'zomb-eh'" + Environment.NewLine + " WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
" AND NotFound = 0" + Environment.NewLine + " AND NotFound = 0" + Environment.NewLine +
" AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine; refreshCooldownClause;
sql = "WITH PostsWithCount AS" + Environment.NewLine + sql = "WITH PostsWithCount AS" + Environment.NewLine +
"(" + Environment.NewLine + "(" + Environment.NewLine +
URLNotesGrabberCORE/Program.cs
+13 -6
View File
@@ -322,7 +322,12 @@ namespace URLNotesGrabberCORE
managedCollectRun = true; managedCollectRun = true;
} }
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName).GetAwaiter().GetResult(); if (forceIgnoreCooldown)
Console.WriteLine(withoutNotesOnly
? "--force: ignoring the periodic re-collect cooldown; already-collected posts in scope are re-queued now"
: "--force: no effect in mode 0 - a full re-check already re-collects every post");
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown).GetAwaiter().GetResult();
break; break;
case "--blogsR": //collect notes from all posts case "--blogsR": //collect notes from all posts
@@ -454,7 +459,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file"); Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date."); Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only).");
Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file "); Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -466,7 +471,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run."); Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
Console.WriteLine("--force\t (with --likes) Ignore cooldown and refresh every fully-backfilled blog"); Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file"); Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
@@ -1345,15 +1350,17 @@ if (shouldInsert)
// blipping on one post. Past this, skipping post-by-post would just hammer a closed door. // blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
const int MaxConsecutiveTransient = 10; const int MaxConsecutiveTransient = 10;
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null) static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false)
{ {
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName); List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown);
if (posts.Count == 0 && !string.IsNullOrWhiteSpace(blogName)) if (posts.Count == 0 && !string.IsNullOrWhiteSpace(blogName))
{ {
// BlogName is matched exactly, so a typo or a case mismatch looks identical to "nothing // BlogName is matched exactly, so a typo or a case mismatch looks identical to "nothing
// left to collect". Say so rather than reporting a silent, instant success. // left to collect". Say so rather than reporting a silent, instant success.
Console.WriteLine($"No posts to collect for blog '{blogName}'. Either it is fully collected, or the name does not match a stored blog (the match is case-sensitive)."); Console.WriteLine($"No posts to collect for blog '{blogName}'. Either it is fully collected, or the name does not match a stored blog (the match is case-sensitive).");
if (withoutNotesOnly && !ignoreRefreshCooldown)
Console.WriteLine("Already-collected posts are re-queued only once their cooldown elapses; add --force to re-collect them now.");
return 0; return 0;
} }
@@ -1451,7 +1458,7 @@ if (shouldInsert)
} }
// Re-fetch the updated list after processing the current post // Re-fetch the updated list after processing the current post
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName); posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown);
} }
} }