feat(collect): add an optional blog filter to --collect
"--collect 1 zomb-eh" now restricts the run to a single blog. The name is bound as a SQLite parameter and matched exactly against Posts.BlogName, which leads the primary key, so the predicate uses that index. Trailing arguments are scanned rather than positionally fixed: a token that parses as a date is the cutoff, anything else is the blog name, in either order. --blog=name forces the blog reading for a name that would otherwise parse as a date. The filter is a pure filter -- the zomb-eh 3-day refresh branch reads from the already-scoped PostsWithCount CTE, so a filtered worklist is a strict subset of the unfiltered one. Verified against TL.db: --collect 1 returns 651 posts, of which --collect 1 zomb-eh returns exactly its 164 and --collect 1 cs1d3blog exactly its 18. Blog-scoped runs are excluded from managed-run state, so collecting one blog cannot mark a full re-check complete. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Claude Opus 5
@@ -855,12 +855,16 @@ namespace URLNotesGrabberCORE
|
||||
/// <param name="withoutNotesOnly"></param>
|
||||
/// <param name="DBPath"></param>
|
||||
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
|
||||
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
|
||||
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, string? DBPath = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
|
||||
|
||||
// A blog filter matches BlogName exactly: the column is BINARY-collated and leads the
|
||||
// Posts primary key, so "= @blogName" rides that index instead of scanning 1.18M rows.
|
||||
bool filterByBlog = !string.IsNullOrWhiteSpace(blogName);
|
||||
|
||||
try
|
||||
{
|
||||
connection.Open();
|
||||
@@ -876,31 +880,17 @@ namespace URLNotesGrabberCORE
|
||||
beforeDateFilter = $"WHERE (U.NotesGatheredDateTime < {unixTimestamp} OR U.NotesGatheredDateTime IS NULL)" + Environment.NewLine;
|
||||
}
|
||||
|
||||
sql = "WITH PostsWithCount AS" + Environment.NewLine +
|
||||
"(" + Environment.NewLine +
|
||||
" SELECT " + Environment.NewLine +
|
||||
" P.BlogName," + Environment.NewLine +
|
||||
" P.PostID," + Environment.NewLine +
|
||||
" 1925013599 AS LatestNoteTimestamp," + Environment.NewLine +
|
||||
" P.NotesGatheredDateTime," + Environment.NewLine +
|
||||
" COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT," + Environment.NewLine +
|
||||
" P.HasNotesGathered," + Environment.NewLine +
|
||||
" P.NotFound," + Environment.NewLine +
|
||||
" P.PostDate" + Environment.NewLine +
|
||||
" FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
|
||||
")," + Environment.NewLine +
|
||||
"Unioned AS" + Environment.NewLine +
|
||||
"(" + Environment.NewLine +
|
||||
" SELECT " + Environment.NewLine +
|
||||
" BlogName," + Environment.NewLine +
|
||||
" PostID," + Environment.NewLine +
|
||||
" LatestNoteTimestamp," + Environment.NewLine +
|
||||
" NotesGatheredDateTime," + Environment.NewLine +
|
||||
" CNT," + Environment.NewLine +
|
||||
" PostDate" + Environment.NewLine +
|
||||
" FROM PostsWithCount" + Environment.NewLine +
|
||||
" WHERE NotFound = 0" + Environment.NewLine +
|
||||
" AND HasNotesGathered = 0" + Environment.NewLine +
|
||||
// Same WHERE/AND juggling WhereIsActive does, extended to the optional blog predicate:
|
||||
// either clause may be absent, so the first one present has to open the WHERE.
|
||||
string sourceClause = AndIsActive("Posts", "P", DBPath) + (filterByBlog ? " AND P.BlogName = @blogName" : string.Empty);
|
||||
string sourceFilter = sourceClause.Length == 0 ? string.Empty : " WHERE" + sourceClause.Substring(" AND".Length);
|
||||
|
||||
// The zomb-eh branch re-queues that blog's *already collected* posts every 3 days. It needs
|
||||
// no blog-filter handling of its own: it reads PostsWithCount, which the filter has already
|
||||
// scoped, so it contributes its rows when the filter names zomb-eh and nothing otherwise.
|
||||
// That keeps a filtered worklist a strict subset of the unfiltered one -- "--collect 1 X"
|
||||
// returns exactly the rows "--collect 1" would have returned for X.
|
||||
string refreshBranch =
|
||||
"" + Environment.NewLine +
|
||||
" UNION " + Environment.NewLine +
|
||||
"" + Environment.NewLine +
|
||||
@@ -914,7 +904,34 @@ namespace URLNotesGrabberCORE
|
||||
" FROM PostsWithCount" + Environment.NewLine +
|
||||
" WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
|
||||
" AND NotFound = 0" + Environment.NewLine +
|
||||
" AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine +
|
||||
" AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine;
|
||||
|
||||
sql = "WITH PostsWithCount AS" + Environment.NewLine +
|
||||
"(" + Environment.NewLine +
|
||||
" SELECT " + Environment.NewLine +
|
||||
" P.BlogName," + Environment.NewLine +
|
||||
" P.PostID," + Environment.NewLine +
|
||||
" 1925013599 AS LatestNoteTimestamp," + Environment.NewLine +
|
||||
" P.NotesGatheredDateTime," + Environment.NewLine +
|
||||
" COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT," + Environment.NewLine +
|
||||
" P.HasNotesGathered," + Environment.NewLine +
|
||||
" P.NotFound," + Environment.NewLine +
|
||||
" P.PostDate" + Environment.NewLine +
|
||||
" FROM Posts P" + sourceFilter + Environment.NewLine +
|
||||
")," + Environment.NewLine +
|
||||
"Unioned AS" + Environment.NewLine +
|
||||
"(" + Environment.NewLine +
|
||||
" SELECT " + Environment.NewLine +
|
||||
" BlogName," + Environment.NewLine +
|
||||
" PostID," + Environment.NewLine +
|
||||
" LatestNoteTimestamp," + Environment.NewLine +
|
||||
" NotesGatheredDateTime," + Environment.NewLine +
|
||||
" CNT," + Environment.NewLine +
|
||||
" PostDate" + Environment.NewLine +
|
||||
" FROM PostsWithCount" + Environment.NewLine +
|
||||
" WHERE NotFound = 0" + Environment.NewLine +
|
||||
" AND HasNotesGathered = 0" + Environment.NewLine +
|
||||
refreshBranch +
|
||||
")" + Environment.NewLine +
|
||||
"SELECT" + Environment.NewLine +
|
||||
" U.BlogName," + Environment.NewLine +
|
||||
@@ -944,6 +961,11 @@ namespace URLNotesGrabberCORE
|
||||
" ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
|
||||
"WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
|
||||
|
||||
if (filterByBlog)
|
||||
{
|
||||
sql += " AND Posts.BlogName = @blogName " + Environment.NewLine;
|
||||
}
|
||||
|
||||
if (beforeDate.HasValue)
|
||||
{
|
||||
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
||||
@@ -963,6 +985,9 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||
{
|
||||
if (filterByBlog)
|
||||
command.Parameters.AddWithValue("@blogName", blogName);
|
||||
|
||||
using (SQLiteDataReader reader = command.ExecuteReader())
|
||||
{
|
||||
while (reader.Read())
|
||||
|
||||
Reference in New Issue
Block a user