Merge branch 'claude/session-fa480e' into master

fix(collect): exclude posts from IsActive=0 blogs in --collect 1 Co-Authored-By: Claude Sonnet 5 <[email protected]>
fix(collect): exclude posts from IsActive=0 blogs in --collect 1
2026-09-03 14:35:57 -05:00 · 2026-09-03 14:34:57 -05:00 · 2026-08-25 12:51:32 -05:00 · 2026-08-24 14:42:43 -05:00 · 2026-08-23 04:34:50 -05:00 · 2026-08-23 04:34:45 -05:00
5 changed files with 256 additions and 140 deletions
@@ -31,7 +31,7 @@ dotnet run -- --test [blogname] [postID]  # Test API for specific post
 - `--test [blogname] [postID]`: Test API note collection
 - `--posts`: Export post blogs to file
 - `--blogs`: Export blog list to file
- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`
+- `--collect [0|1] [datetime] [blogname]`: Collect notes for posts in DB. Optional `blogname` restricts the run to one blog (exact match), e.g. `--collect 1 zomb-eh`. Add `--force` to ignore the periodic re-collect cooldown so already-collected posts are re-queued immediately (mode 1 only). Add `--fromDate <datetime>` / `--toDate <datetime>` to only re-queue already-collected posts whose original PostDate is on/after / on/before that date (mode 1 only; either or both may be given; applies with or without `--force`)
 - `--blogsR`: Export reply blogs to file
 - `--blogsO [start] [stop]`: Export blogs within range

@@ -1,100 +1,109 @@
-<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="4486"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd0000000100000002000005470000029afc0100000006fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000000005470000011100fffffffb000000160064006f0063006b00420072006f00770073006500340100000000000005f40000000000000000000005470000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="ApiKeyPoolMeta" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="29"/><column index="2" value="64"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="7" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="108"/><column index="3" value="63"/><column index="4" value="156"/><column index="5" value="60"/><column index="6" value="81"/><column index="7" value="85"/><column index="8" value="156"/><column index="9" value="156"/><column index="10" value="151"/><column index="11" value="125"/><column index="12" value="129"/></column_widths><filter_values><column index="4" value="1"/><column index="7" value="&gt;2026-05-27 17:22:36"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="198"/><column index="2" value="144"/><column index="3" value="251"/><column index="4" value="84"/><column index="5" value="53"/><column index="6" value="300"/><column index="7" value="116"/><column index="8" value="152"/><column index="9" value="89"/><column index="10" value="63"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="14" mode="1"/></sort><column_widths><column index="1" value="236"/><column index="2" value="144"/><column index="3" value="126"/><column index="4" value="32"/><column index="5" value="32"/><column index="6" value="32"/><column index="7" value="32"/><column index="8" value="32"/><column index="9" value="0"/><column index="10" value="0"/><column index="11" value="0"/><column index="12" value="243"/><column index="13" value="300"/><column index="14" value="53"/><column index="15" value="37351"/><column index="16" value="300"/><column index="17" value="41"/><column index="18" value="75"/><column index="19" value="96"/><column index="20" value="300"/><column index="21" value="96"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="548"/><column index="25" value="60"/><column index="26" value="213"/><column index="27" value="532"/><column index="28" value="152"/><column index="29" value="152"/><column index="30" value="69"/><column index="31" value="63"/></column_widths><filter_values><column index="2" value="0"/><column index="1" value="734568371821084672"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns><column index="9" value="1"/><column index="10" value="1"/><column index="11" value="1"/></hidden_columns><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
-SET HasNotesGathered = 0
-WHERE (BlogName, PostID) IN (
-    SELECT p.BlogName, p.PostID
-    FROM Posts p
-    WHERE p.HasNotesGathered = 1
-	AND P.notesGatheredDatetime &lt; 1774294520
-    AND EXISTS (
-        SELECT 1 
-        FROM Notes n 
-        WHERE n.PostID = p.PostID 
-          AND n.RootBlogName = p.BlogName
-          --AND n.Type NOT IN ('reblog', 'reply')
-    )
-    ORDER BY P.PostDate ASC
-    --LIMIT 500
-);</sql><sql name="Mark Blogs">select *
-from Blogs
--update blogs set HasBeenOutput = 1
-where HasBeenOutput = 0
-AND
-blogname in
-(
-'teaberrybee',
-'reddevilgoddesstoo',
-'waywardog13',
-'wzjustbrowsing-blog',
-'lewerta',
-'nudenymph',
-'caylachief'
-
-)</sql><sql name="New Notes">select P.slug, N.replyText, n.RootBlogName, n.PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, n.RootBlogName || '.tumblr.com/post/' || n.postid, datetime(timestamp, 'unixepoch')
-from Notes N inner join Posts P on p.PostID = n.PostID
-where 
-DatetimeCrawled &gt; '2026-08-07 11:47:22' and type like 'r%'
-and P.IsActive = 1 
-order by n.DatetimeCrawled</sql><sql name="Pull Blogs">SELECT distinct
-	'''' || blogname || ''',',
-	blogs.*
-	, blogname || '.tumblr.com'
-FROM
-	Blogs
-		inner JOIN
-	Notes on notes.noteBlogName = blogs.BlogName
-WHERE
-	HasBeenOutput = 0 and type = 'reblog'
-order by
-	Notes.Type desc,
-	DateAdded desc
-LIMIT 100;</sql><sql name="SQL 7">WITH ReplyCounts AS (
-    SELECT 
-        NoteBlogName, 
-        COUNT(DISTINCT replyText) AS DistinctReplyCount
-    FROM Notes
-	where replyText &lt;&gt; '.'
-    GROUP BY NoteBlogName
-)
-SELECT 
-    n.RootBlogName || '.tumblr.com/post/' || n.PostID AS PostURL, postid,
-    n.NoteBlogName,
-    n.replyText,
-    c.DistinctReplyCount
-FROM Notes n
-JOIN ReplyCounts c ON n.NoteBlogName = c.NoteBlogName
-where replyText &lt;&gt; '.' and type &lt;&gt; 'reply'
--AND N.NoteBlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592',
--'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' )
-order by c.DistinctReplyCount desc, n.NoteBlogName, n.DateModified desc, replyText, RootBlogName, PostID</sql><sql name="Collect">WITH PostsWithCount AS ( SELECT P.BlogName, P.PostID, 1925013599 AS LatestNoteTimestamp, P.NotesGatheredDateTime, COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT, P.HasNotesGathered, P.NotFound, P.PostDate FROM Posts P WHERE COALESCE(P.IsActive, 1) = 1 ), Unioned AS ( SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE NotFound = 0 AND HasNotesGathered = 0 UNION SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE BlogName = 'zomb-eh' AND NotFound = 0 AND NotesGatheredDateTime &lt; unixepoch('now', 'localtime', '-3 days') ) SELECT U.BlogName, U.PostID, U.LatestNoteTimestamp, U.NotesGatheredDateTime, U.CNT FROM Unioned U WHERE (U.NotesGatheredDateTime &lt; 1786134037 OR U.NotesGatheredDateTime IS NULL) ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;</sql><sql name="Del Posts">delete from posts where postid in 
-(
-'741662499571728384',
-178892849664,
-178264721139,
-177012868749,
-169950081964,
-755440787056099328
-)</sql><sql name="notes NO post*">select *
-- delete
-from notes
-where postid not in (select distinct postid from posts where IsActive = 1)</sql><sql name="SQL 9">SELECT	
-	*
-FROM	
-	POSTS P
-WHERE	
-	P.ByLikes = 1
-	AND
-	P.DateCreated &gt; '2026-05-26 17:47:32'
-ORDER BY
-	P.DateCreated desc</sql><sql name="SQL 13">update posts set IsActive = 0 where blogname IN ( 'shoebiedoo', 'redheaded-girlygirl', 'xlittle-ghost' )</sql><sql name="SQL 14*">update Posts␍
-set IsActive = 0␍
-where postid in␍
-(␍
-␍
-␍
-'731937314675310592'␍
-␍
-␍
-␍
-)␍
-␍
-</sql><current_tab id="7"/></tab_sql></sqlb_project>
+<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="4486"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd0000000100000002000005470000029afc0100000006fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000000005470000011100fffffffb000000160064006f0063006b00420072006f00770073006500340100000000000005f40000000000000000000005470000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="ApiKeyPoolMeta" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="29"/><column index="2" value="64"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="7" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="108"/><column index="3" value="63"/><column index="4" value="156"/><column index="5" value="60"/><column index="6" value="81"/><column index="7" value="85"/><column index="8" value="156"/><column index="9" value="156"/><column index="10" value="151"/><column index="11" value="125"/><column index="12" value="129"/></column_widths><filter_values><column index="4" value="1"/><column index="7" value="&gt;2026-05-27 17:22:36"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="198"/><column index="2" value="144"/><column index="3" value="251"/><column index="4" value="84"/><column index="5" value="53"/><column index="6" value="300"/><column index="7" value="116"/><column index="8" value="152"/><column index="9" value="89"/><column index="10" value="63"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="14" mode="1"/></sort><column_widths><column index="1" value="236"/><column index="2" value="144"/><column index="3" value="126"/><column index="4" value="32"/><column index="5" value="32"/><column index="6" value="32"/><column index="7" value="32"/><column index="8" value="32"/><column index="9" value="0"/><column index="10" value="0"/><column index="11" value="0"/><column index="12" value="243"/><column index="13" value="300"/><column index="14" value="53"/><column index="15" value="37351"/><column index="16" value="300"/><column index="17" value="41"/><column index="18" value="75"/><column index="19" value="96"/><column index="20" value="300"/><column index="21" value="96"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="548"/><column index="25" value="60"/><column index="26" value="213"/><column index="27" value="532"/><column index="28" value="152"/><column index="29" value="152"/><column index="30" value="69"/><column index="31" value="63"/></column_widths><filter_values><column index="2" value="0"/><column index="1" value="734568371821084672"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns><column index="9" value="1"/><column index="10" value="1"/><column index="11" value="1"/></hidden_columns><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
+SET HasNotesGathered = 0
+WHERE (BlogName, PostID) IN (
+    SELECT p.BlogName, p.PostID
+    FROM Posts p
+    WHERE p.HasNotesGathered = 1
+	AND P.notesGatheredDatetime &lt; 1774294520
+    AND EXISTS (
+        SELECT 1 
+        FROM Notes n 
+        WHERE n.PostID = p.PostID 
+          AND n.RootBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = p.BlogName)
+          --AND n.TypeId NOT IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply'))
+    )
+    ORDER BY P.PostDate ASC
+    --LIMIT 500
+);</sql><sql name="Mark Blogs">select *
+from Blogs
+--update blogs set HasBeenOutput = 1
+where HasBeenOutput = 0
+AND
+blogname in
+(
+'teaberrybee',
+'reddevilgoddesstoo',
+'waywardog13',
+'wzjustbrowsing-blog',
+'lewerta',
+'nudenymph',
+'caylachief'
+
+)</sql><sql name="New Notes">select P.slug, N.replyText, rbn.BlogName as RootBlogName, n.PostID, nbn.BlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, nt.Type, rbn.BlogName || '.tumblr.com/post/' || n.postid, datetime(timestamp, 'unixepoch')
+from Notes N
+inner join Posts P on p.PostID = n.PostID
+inner join BlogNames rbn on rbn.BlogId = n.RootBlogId
+inner join BlogNames nbn on nbn.BlogId = n.NoteBlogId
+inner join NoteTypes nt on nt.TypeId = n.TypeId
+where
+DatetimeCrawled &gt; '2026-08-07 11:47:22' and nt.Type like 'r%'
+and P.IsActive = 1
+order by n.DatetimeCrawled</sql><sql name="Pull Blogs">SELECT distinct
+	'''' || blogname || ''',',
+	blogs.*
+	, blogname || '.tumblr.com'
+FROM
+	Blogs
+		inner JOIN
+	Notes on notes.noteBlogId = blogs.BlogId
+		inner JOIN
+	NoteTypes on NoteTypes.TypeId = Notes.TypeId
+WHERE
+	HasBeenOutput = 0 and NoteTypes.Type = 'reblog'
+order by
+	NoteTypes.Type desc,
+	DateAdded desc
+LIMIT 100;</sql><sql name="SQL 7">WITH ReplyCounts AS (
+    SELECT
+        NoteBlogId,
+        COUNT(DISTINCT replyText) AS DistinctReplyCount
+    FROM Notes
+	where replyText &lt;&gt; '.'
+    GROUP BY NoteBlogId
+)
+SELECT
+    rbn.BlogName || '.tumblr.com/post/' || n.PostID AS PostURL, postid,
+    nbn.BlogName AS NoteBlogName,
+    n.replyText,
+    c.DistinctReplyCount
+FROM Notes n
+JOIN ReplyCounts c ON n.NoteBlogId = c.NoteBlogId
+JOIN BlogNames rbn ON rbn.BlogId = n.RootBlogId
+JOIN BlogNames nbn ON nbn.BlogId = n.NoteBlogId
+JOIN NoteTypes t ON t.TypeId = n.TypeId
+where replyText &lt;&gt; '.' and t.Type &lt;&gt; 'reply'
+--AND nbn.BlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592',
+--'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' )
+order by c.DistinctReplyCount desc, nbn.BlogName, n.DateModified desc, replyText, rbn.BlogName, PostID</sql><sql name="Collect">WITH PostsWithCount AS ( SELECT P.BlogName, P.PostID, 1925013599 AS LatestNoteTimestamp, P.NotesGatheredDateTime, COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT, P.HasNotesGathered, P.NotFound, P.PostDate FROM Posts P WHERE COALESCE(P.IsActive, 1) = 1 ), Unioned AS ( SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE NotFound = 0 AND HasNotesGathered = 0 UNION SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE BlogName = 'zomb-eh' AND NotFound = 0 AND NotesGatheredDateTime &lt; unixepoch('now', 'localtime', '-3 days') ) SELECT U.BlogName, U.PostID, U.LatestNoteTimestamp, U.NotesGatheredDateTime, U.CNT FROM Unioned U WHERE (U.NotesGatheredDateTime &lt; 1786134037 OR U.NotesGatheredDateTime IS NULL) ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;</sql><sql name="Del Posts">delete from posts where postid in 
+(
+'741662499571728384',
+178892849664,
+178264721139,
+177012868749,
+169950081964,
+755440787056099328
+)</sql><sql name="notes NO post*">select *
+-- delete
+from notes
+where postid not in (select distinct postid from posts where IsActive = 1)</sql><sql name="SQL 9">SELECT	
+	*
+FROM	
+	POSTS P
+WHERE	
+	P.ByLikes = 1
+	AND
+	P.DateCreated &gt; '2026-05-26 17:47:32'
+ORDER BY
+	P.DateCreated desc</sql><sql name="SQL 13">update posts set IsActive = 0 where blogname IN ( 'shoebiedoo', 'redheaded-girlygirl', 'xlittle-ghost' )</sql><sql name="SQL 14*">update Posts
+set IsActive = 0
+where postid in
+(
+
+
+'731937314675310592'
+
+
+
+)
+
+</sql><current_tab id="7"/></tab_sql></sqlb_project>
@@ -1,24 +1,30 @@
-<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="" readonly="1" foreign_keys="" case_sensitive_like="" temp_store="" wal_autocheckpoint="" synchronous=""/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/></tab_structure><tab_browse><table title="." custom_title="0" dock_id="4" table="0,0:"/><dock_state state="000000ff00000000fd0000000100000002000005f40000030ffc0100000002fb000000160064006f0063006b00420072006f00770073006500310100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011700ffffff000005f40000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings/></tab_browse><tab_sql><sql name="SQL 1">SELECT	
-	BlogName || '.tumblr.com/post/' || postID,
-	
-	datetime(NotesGatheredDateTime, 'unixepoch'), *
-FROM
-	Posts
-WHERE
-	NotesGatheredDateTime &lt;&gt; 0
-ORDER BY
-	postdate desc</sql><sql name="SQL 2*">SELECT	
-	datetime(TimeStamp, 'unixepoch'),
-	RootBlogName || '.tumblr.com/post/' || N.postid,
-	*,
-	NoteBlogName || '.tumblr.com'
-FROM
-	Notes N␍
-	inner JOIN␍
-	Posts P on P.PostID = N.PostID and P.BlogName = N.RootBlogName
-WHERE	RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')␍
-		and type like 'r%'␍
-		and RootBlogName = 'zomb-eh'␍
-		and P.HasImage = 1
-ORDER BY
-	TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project>
+<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="" readonly="1" foreign_keys="" case_sensitive_like="" temp_store="" wal_autocheckpoint="" synchronous=""/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/></tab_structure><tab_browse><table title="." custom_title="0" dock_id="4" table="0,0:"/><dock_state state="000000ff00000000fd0000000100000002000005f40000030ffc0100000002fb000000160064006f0063006b00420072006f00770073006500310100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011700ffffff000005f40000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings/></tab_browse><tab_sql><sql name="SQL 1">SELECT	
+	BlogName || '.tumblr.com/post/' || postID,
+	
+	datetime(NotesGatheredDateTime, 'unixepoch'), *
+FROM
+	Posts
+WHERE
+	NotesGatheredDateTime &lt;&gt; 0
+ORDER BY
+	postdate desc</sql><sql name="SQL 2*">SELECT
+	datetime(TimeStamp, 'unixepoch'),
+	rbn.BlogName || '.tumblr.com/post/' || N.postid,
+	*,
+	nbn.BlogName || '.tumblr.com'
+FROM
+	Notes N
+	inner JOIN
+	BlogNames rbn on rbn.BlogId = N.RootBlogId
+	inner JOIN
+	BlogNames nbn on nbn.BlogId = N.NoteBlogId
+	inner JOIN
+	NoteTypes t on t.TypeId = N.TypeId
+	inner JOIN
+	Posts P on P.PostID = N.PostID and P.BlogName = rbn.BlogName
+WHERE	rbn.BlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')
+		and t.Type like 'r%'
+		and rbn.BlogName = 'zomb-eh'
+		and P.HasImage = 1
+ORDER BY
+	TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project>
@@ -855,12 +855,15 @@ namespace URLNotesGrabberCORE
        #region Gets

        /// <summary>
-        /// 
+        ///
        /// </summary>
        /// <param name="withoutNotesOnly"></param>
+        /// <param name="ignoreRefreshCooldown">Drops the age gate on the periodic re-queue branch (--force).</param>
+        /// <param name="fromDate">Lower bound on the *original post's* PostDate for the periodic re-queue branch (--fromDate). Independent of ignoreRefreshCooldown -- applies whether or not --force is also given.</param>
+        /// <param name="toDate">Upper bound on the *original post's* PostDate for the periodic re-queue branch (--toDate). Same independence from ignoreRefreshCooldown as fromDate.</param>
        /// <param name="DBPath"></param>
        /// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
-        public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, string? DBPath = null)
+        public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null, DateTime? toDate = null, string? DBPath = null)
        {
            DBPath ??= GetDefaultDbPath();
            using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -885,16 +888,48 @@ namespace URLNotesGrabberCORE
                        beforeDateFilter = $"WHERE (U.NotesGatheredDateTime < {unixTimestamp} OR U.NotesGatheredDateTime IS NULL)" + Environment.NewLine;
                    }

+                    // Blogs.IsActive is the crawler's work-selection flag (Rolodex removal sets it to 0)
+                    // and is independent of Posts.IsActive/Notes.IsActive -- deactivating a blog does not
+                    // touch its posts' own IsActive column. AndIsActive("Posts", ...) above therefore does
+                    // not catch a deactivated blog; this join against the source rows is what does, so a
+                    // blog taken IsActive = 0 in Blogs stops being re-queued by --collect 1 even if its
+                    // posts were never individually marked inactive. Blogs.BlogName is that table's PRIMARY
+                    // KEY, so the join rides an index rather than scanning it.
+                    //
                    // Same WHERE/AND juggling WhereIsActive does, extended to the optional blog predicate:
                    // either clause may be absent, so the first one present has to open the WHERE.
-                    string sourceClause = AndIsActive("Posts", "P", DBPath) + (filterByBlog ? " AND P.BlogName = @blogName" : string.Empty);
+                    string sourceClause = AndIsActive("Posts", "P", DBPath) + " AND COALESCE(BL.IsActive, 1) = 1" + (filterByBlog ? " AND P.BlogName = @blogName" : string.Empty);
                    string sourceFilter = sourceClause.Length == 0 ? string.Empty : " WHERE" + sourceClause.Substring(" AND".Length);

                    // The zomb-eh branch re-queues that blog's *already collected* posts every 3 days. It needs
-                    // no blog-filter handling of its own: it reads PostsWithCount, which the filter has already
-                    // scoped, so it contributes its rows when the filter names zomb-eh and nothing otherwise.
-                    // That keeps a filtered worklist a strict subset of the unfiltered one -- "--collect 1 X"
-                    // returns exactly the rows "--collect 1" would have returned for X.
+                    // no blog-filter or IsActive handling of its own: it reads PostsWithCount, which the source
+                    // filter above -- Blogs.IsActive included -- has already scoped, so it contributes its rows
+                    // only when zomb-eh itself is still IsActive = 1 there. That keeps a filtered worklist a
+                    // strict subset of the unfiltered one -- "--collect 1 X" returns exactly the rows
+                    // "--collect 1" would have returned for X.
+                    //
+                    // --force drops the age gate only. NotFound = 0 and the IsActive/blog scoping above still
+                    // apply: the flag is "re-collect early", not "collect rows every other path excludes".
+                    string refreshCooldownClause = ignoreRefreshCooldown
+                        ? string.Empty
+                        : "      AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine;
+
+                    // --fromDate bounds the *original post's* PostDate, not the re-collect cooldown --
+                    // it stacks with refreshCooldownClause instead of replacing it, so it applies the
+                    // same way whether or not --force also dropped the cooldown. A NULL PostDate never
+                    // satisfies ">=" and is excluded, same as an unfiltered run would still include it
+                    // (there's nothing to compare here, so this only narrows, never widens, the result).
+                    string fromDateClause = fromDate.HasValue
+                        ? "      AND PostDate >= @fromDate" + Environment.NewLine
+                        : string.Empty;
+
+                    // --toDate is the same deal, mirrored: stacks alongside fromDateClause/
+                    // refreshCooldownClause rather than replacing either, so --fromDate and --toDate
+                    // can be given together (or alone) and both hold with or without --force.
+                    string toDateClause = toDate.HasValue
+                        ? "      AND PostDate <= @toDate" + Environment.NewLine
+                        : string.Empty;
+
                    string refreshBranch =
                          "" + Environment.NewLine +
                          "    UNION " + Environment.NewLine +
@@ -909,7 +944,9 @@ namespace URLNotesGrabberCORE
                          "    FROM PostsWithCount" + Environment.NewLine +
                          "    WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
                          "      AND NotFound = 0" + Environment.NewLine +
-                          "      AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine;
+                          refreshCooldownClause +
+                          fromDateClause +
+                          toDateClause;

                    sql = "WITH PostsWithCount AS" + Environment.NewLine +
                          "(" + Environment.NewLine +
@@ -922,7 +959,7 @@ namespace URLNotesGrabberCORE
                          "        P.HasNotesGathered," + Environment.NewLine +
                          "        P.NotFound," + Environment.NewLine +
                          "        P.PostDate" + Environment.NewLine +
-                          "    FROM Posts P" + sourceFilter + Environment.NewLine +
+                          "    FROM Posts P LEFT JOIN Blogs BL ON BL.BlogName = P.BlogName" + sourceFilter + Environment.NewLine +
                          ")," + Environment.NewLine +
                          "Unioned AS" + Environment.NewLine +
                          "(" + Environment.NewLine +
@@ -993,6 +1030,13 @@ namespace URLNotesGrabberCORE
                    if (filterByBlog)
                        command.Parameters.AddWithValue("@blogName", blogName);

+                    // Only ever referenced by the zomb-eh refresh branch, which only exists when
+                    // withoutNotesOnly is true -- harmless to bind unconditionally otherwise.
+                    if (fromDate.HasValue)
+                        command.Parameters.AddWithValue("@fromDate", fromDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
+                    if (toDate.HasValue)
+                        command.Parameters.AddWithValue("@toDate", toDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
+
                    using (SQLiteDataReader reader = command.ExecuteReader())
                    {
                        while (reader.Read())
@@ -44,6 +44,8 @@ namespace URLNotesGrabberCORE
            bool apiExplicitlySet = false;
            string startFromBlogName = string.Empty;
            bool forceIgnoreCooldown = false;
+            DateTime? fromDate = null;
+            DateTime? toDate = null;
            List<string> filteredArgs = new List<string>();
            for (int i = 0; i < args.Length; i++)
            {
@@ -60,6 +62,34 @@ namespace URLNotesGrabberCORE
                    continue;
                }

+                if (string.Equals(args[i], "--fromDate", StringComparison.OrdinalIgnoreCase))
+                {
+                    if (i + 1 < args.Length && DateTime.TryParse(args[i + 1], out DateTime parsedFromDate))
+                    {
+                        fromDate = parsedFromDate;
+                        i++;
+                    }
+                    else
+                    {
+                        Console.WriteLine("--Missing or unparseable date after --fromDate. Ignoring.--");
+                    }
+                    continue;
+                }
+
+                if (string.Equals(args[i], "--toDate", StringComparison.OrdinalIgnoreCase))
+                {
+                    if (i + 1 < args.Length && DateTime.TryParse(args[i + 1], out DateTime parsedToDate))
+                    {
+                        toDate = parsedToDate;
+                        i++;
+                    }
+                    else
+                    {
+                        Console.WriteLine("--Missing or unparseable date after --toDate. Ignoring.--");
+                    }
+                    continue;
+                }
+
                if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
                {
                    apiSectionName = "TumblrApi3";
@@ -148,6 +178,12 @@ namespace URLNotesGrabberCORE
            if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
            {
                int postsAdded = 0;
+                // This is the mode that actually gets run day to day, so the schema migration and
+                // the PostType backfill have to happen here too. They used to hang off --ingest,
+                // --output and friends only, which meant the untyped rows this traversal creates
+                // could sit unrepaired indefinitely while the one command everyone runs skipped
+                // the fix entirely. Idempotent, so paying it on every run costs nothing.
+                DataAccess.EnsureTTFileHelperColumnsExist();
                try
                {
                    DataAccess.EnableImportModePragmas();
@@ -316,7 +352,22 @@ namespace URLNotesGrabberCORE
                            managedCollectRun = true;
                        }

-                        exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName).GetAwaiter().GetResult();
+                        if (forceIgnoreCooldown)
+                            Console.WriteLine(withoutNotesOnly
+                                ? "--force: ignoring the periodic re-collect cooldown; already-collected posts in scope are re-queued now"
+                                : "--force: no effect in mode 0 - a full re-check already re-collects every post");
+
+                        if (fromDate.HasValue)
+                            Console.WriteLine(withoutNotesOnly
+                                ? $"--fromDate: only re-queuing already-collected posts originally posted on/after {fromDate.Value} (applies with or without --force)"
+                                : "--fromDate: no effect in mode 0 - it only bounds the periodic re-queue branch");
+
+                        if (toDate.HasValue)
+                            Console.WriteLine(withoutNotesOnly
+                                ? $"--toDate: only re-queuing already-collected posts originally posted on/before {toDate.Value} (applies with or without --force)"
+                                : "--toDate: no effect in mode 0 - it only bounds the periodic re-queue branch");
+
+                        exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun, collectBlogName, forceIgnoreCooldown, fromDate, toDate).GetAwaiter().GetResult();
                        break;

                    case "--blogsR": //collect notes from all posts
@@ -448,7 +499,7 @@ namespace URLNotesGrabberCORE

            Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");

-            Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date.");
+            Console.WriteLine("--collect [0|1] [datetime] [blogname]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking). Optional blogname restricts the run to that blog (exact, case-sensitive match) and also runs as a one-off; e.g. \"--collect 1 zomb-eh\". datetime and blogname may be given in either order - use --blog=name if a blog name would otherwise parse as a date. Add --force to ignore the periodic re-collect cooldown and re-queue already-collected posts immediately (mode 1 only). Add --fromDate <datetime> / --toDate <datetime> to only re-queue already-collected posts originally posted on/after / on/before that date (mode 1 only; either or both may be given; applies with or without --force).");

            Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");

@@ -460,7 +511,11 @@ namespace URLNotesGrabberCORE

            Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");

-            Console.WriteLine("--force\t (with --likes) Ignore cooldown and refresh every fully-backfilled blog");
+            Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
+
+            Console.WriteLine("--fromDate <datetime>\t With --collect 1, only re-queue already-collected posts originally posted on/after <datetime>. Independent of --force - applies whether or not the cooldown is also bypassed.");
+
+            Console.WriteLine("--toDate <datetime>\t With --collect 1, only re-queue already-collected posts originally posted on/before <datetime>. Independent of --force; may be combined with --fromDate for a range.");

            Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");

@@ -1339,15 +1394,17 @@ if (shouldInsert)
        // blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
        const int MaxConsecutiveTransient = 10;

-        static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null)
+        static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false, string? blogName = null, bool ignoreRefreshCooldown = false, DateTime? fromDate = null, DateTime? toDate = null)
        {
-            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName);
+            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate, toDate);

            if (posts.Count == 0 && !string.IsNullOrWhiteSpace(blogName))
            {
                // BlogName is matched exactly, so a typo or a case mismatch looks identical to "nothing
                // left to collect". Say so rather than reporting a silent, instant success.
                Console.WriteLine($"No posts to collect for blog '{blogName}'. Either it is fully collected, or the name does not match a stored blog (the match is case-sensitive).");
+                if (withoutNotesOnly && !ignoreRefreshCooldown)
+                    Console.WriteLine("Already-collected posts are re-queued only once their cooldown elapses; add --force to re-collect them now.");
                return 0;
            }

@@ -1445,7 +1502,7 @@ if (shouldInsert)
                        }

                        // Re-fetch the updated list after processing the current post
-                        posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName);
+                        posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate, blogName, ignoreRefreshCooldown, fromDate, toDate);
                    }
                }
Author	SHA1	Message	Date
jimandClaude Sonnet 5	b8231d6a4c	Merge branch 'claude/session-fa480e' into master fix(collect): exclude posts from IsActive=0 blogs in --collect 1 Co-Authored-By: Claude Sonnet 5 <[email protected]>	2026-09-03 14:35:57 -05:00
jimandClaude Sonnet 5	bbf05b3863	fix(collect): exclude posts from IsActive=0 blogs in --collect 1 Blogs.IsActive is the crawler's work-selection flag (Rolodex removal sets it to 0) and is independent of Posts.IsActive/Notes.IsActive -- deactivating a blog never touches its posts' own IsActive column, so --collect 1 kept re-queuing posts for blogs that had been deactivated. Join Blogs into the PostsWithCount CTE's source filter and require COALESCE(BL.IsActive, 1) = 1. Since the hardcoded zomb-eh re-queue branch reads from PostsWithCount rather than Posts directly, it now inherits this filter automatically -- if zomb-eh is ever deactivated, its rows disappear from PostsWithCount and the union branch contributes nothing, with no special-case code needed. Co-Authored-By: Claude Sonnet 5 <[email protected]>	2026-09-03 14:34:57 -05:00
jim	ea2afc9d35	feat(collect): add --toDate upper bound for zomb-eh re-queue branch Mirrors --fromDate: bounds the zomb-eh periodic re-queue branch by PostDate <= the given date. Stacks alongside --fromDate and the cooldown clause rather than replacing either, so --fromDate/--toDate and --force can all be combined, and each applies with or without --force. Named --toDate rather than --end to pair with --fromDate -- --start already exists as an unrelated flag (resume folder traversal at a blog name).	2026-08-25 12:51:32 -05:00
jim	7bf270e47d	feat(collect): add --fromDate lower bound for zomb-eh re-queue branch The zomb-eh periodic re-queue branch in GetPosts had no lower bound on the post's original PostDate -- it re-queued every already-collected zomb-eh post past the 3-day cooldown, regardless of age. Add --fromDate <datetime> to bound that branch by PostDate >= the given date. It stacks with the existing cooldown clause rather than replacing it, so it applies the same way whether or not --force also drops the cooldown.	2026-08-24 14:42:43 -05:00
jim	58d7b1d05e	Merge branch 'claude/collect-command-issue-09fcb8' into master	2026-08-23 04:34:50 -05:00
jimandClaude Opus 5	f41957fd2f	feat(collect): let --force ignore the re-collect cooldown --collect 1 <blog> returned an empty worklist whenever the periodic re-queue branch was still inside its 3-day window, with no way to ask for the re-collect early. GetPosts now composes that age predicate conditionally, and the existing global --force flag - already "ignore cooldown" for --likes - drives it. Only the age gate drops: NotFound = 0, the IsActive filter and the blog scoping still apply. The flag is a no-op for mode 0, which re-collects every post regardless, and says so rather than pretending to act. Co-Authored-By: Claude Opus 5 <[email protected]>	2026-08-23 04:34:45 -05:00
jimandClaude Sonnet 5	a28c5cc9ec	fix(dbbrowser): rewrite saved queries for the Notes integer schema Notes.RootBlogName/NoteBlogName/Type were replaced by RootBlogId/NoteBlogId/ TypeId (resolved via BlogNames and NoteTypes) back on 2026-08-07. The saved DB Browser for SQLite queries in RERUN.sqbpro and TL.sqbpro still referenced the old text columns and failed against the migrated TL.db. Rewrote the 5 affected queries per TL.db.md's porting guide: Notes<->Blogs joins go through Blogs.BlogId in one hop, Notes<->Posts joins route through BlogNames (Posts has no BlogId), and type filters resolve through NoteTypes. Verified read-only against the live TL.db -- all 5 execute without error. Co-Authored-By: Claude Sonnet 5 <[email protected]>	2026-08-20 09:07:08 -05:00
jim	d6a96f7885	Merge branch 'claude/posttype-noargs-backfill' into master	2026-08-20 08:15:54 -05:00
jimandClaude Opus 5	864b468d96	fix(posttype): run the migration and backfill in no-args mode The backfill hung off --ingest, --output, --correct, --importposts and --updatepaths, because those were where EnsureTTFileHelperColumnsExist was already being called. But the no-argument traversal is the mode that actually gets run day to day, and it called none of them -- so the command used most often was the one command that never repaired an untyped row. TraverseDirectory already types the posts it inserts, from the filename it is reading. This closes the other half: the rows already sitting untyped now get fixed by an ordinary run, with no separate maintenance command. Called before BeginImportSession so it uses its own connection rather than contending with the import session's, and before the traversal so existing rows are typed first and newly inserted ones arrive already typed. Co-Authored-By: Claude Opus 5 <[email protected]>	2026-08-20 08:15:54 -05:00
jim	e6a3efba5b	Merge branch 'claude/posttype-unknown-fix' into master	2026-08-20 08:01:01 -05:00