Author SHA1 Message Date
jimandClaude Sonnet 5 a28c5cc9ec fix(dbbrowser): rewrite saved queries for the Notes integer schema
Notes.RootBlogName/NoteBlogName/Type were replaced by RootBlogId/NoteBlogId/
TypeId (resolved via BlogNames and NoteTypes) back on 2026-08-07. The saved
DB Browser for SQLite queries in RERUN.sqbpro and TL.sqbpro still referenced
the old text columns and failed against the migrated TL.db.

Rewrote the 5 affected queries per TL.db.md's porting guide: Notes<->Blogs
joins go through Blogs.BlogId in one hop, Notes<->Posts joins route through
BlogNames (Posts has no BlogId), and type filters resolve through NoteTypes.
Verified read-only against the live TL.db -- all 5 execute without error.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-08-20 09:07:08 -05:00
jim d6a96f7885 Merge branch 'claude/posttype-noargs-backfill' into master 2026-08-20 08:15:54 -05:00
jimandClaude Opus 5 864b468d96 fix(posttype): run the migration and backfill in no-args mode
The backfill hung off --ingest, --output, --correct, --importposts and
--updatepaths, because those were where EnsureTTFileHelperColumnsExist was
already being called. But the no-argument traversal is the mode that
actually gets run day to day, and it called none of them -- so the command
used most often was the one command that never repaired an untyped row.

TraverseDirectory already types the posts it inserts, from the filename it
is reading. This closes the other half: the rows already sitting untyped
now get fixed by an ordinary run, with no separate maintenance command.

Called before BeginImportSession so it uses its own connection rather than
contending with the import session's, and before the traversal so existing
rows are typed first and newly inserted ones arrive already typed.

Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-20 08:15:54 -05:00
3 changed files with 145 additions and 124 deletions
+25 -16
View File
@@ -9,8 +9,8 @@ WHERE (BlogName, PostID) IN (
SELECT 1 SELECT 1
FROM Notes n FROM Notes n
WHERE n.PostID = p.PostID WHERE n.PostID = p.PostID
AND n.RootBlogName = p.BlogName AND n.RootBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = p.BlogName)
--AND n.Type NOT IN ('reblog', 'reply') --AND n.TypeId NOT IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply'))
) )
ORDER BY P.PostDate ASC ORDER BY P.PostDate ASC
--LIMIT 500 --LIMIT 500
@@ -29,10 +29,14 @@ blogname in
'nudenymph', 'nudenymph',
'caylachief' 'caylachief'
)</sql><sql name="New Notes">select P.slug, N.replyText, n.RootBlogName, n.PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, n.RootBlogName || '.tumblr.com/post/' || n.postid, datetime(timestamp, 'unixepoch') )</sql><sql name="New Notes">select P.slug, N.replyText, rbn.BlogName as RootBlogName, n.PostID, nbn.BlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, nt.Type, rbn.BlogName || '.tumblr.com/post/' || n.postid, datetime(timestamp, 'unixepoch')
from Notes N inner join Posts P on p.PostID = n.PostID from Notes N
inner join Posts P on p.PostID = n.PostID
inner join BlogNames rbn on rbn.BlogId = n.RootBlogId
inner join BlogNames nbn on nbn.BlogId = n.NoteBlogId
inner join NoteTypes nt on nt.TypeId = n.TypeId
where where
DatetimeCrawled &gt; '2026-08-07 11:47:22' and type like 'r%' DatetimeCrawled &gt; '2026-08-07 11:47:22' and nt.Type like 'r%'
and P.IsActive = 1 and P.IsActive = 1
order by n.DatetimeCrawled</sql><sql name="Pull Blogs">SELECT distinct order by n.DatetimeCrawled</sql><sql name="Pull Blogs">SELECT distinct
'''' || blogname || ''',', '''' || blogname || ''',',
@@ -41,31 +45,36 @@ order by n.DatetimeCrawled</sql><sql name="Pull Blogs">SELECT distinct
FROM FROM
Blogs Blogs
inner JOIN inner JOIN
Notes on notes.noteBlogName = blogs.BlogName Notes on notes.noteBlogId = blogs.BlogId
inner JOIN
NoteTypes on NoteTypes.TypeId = Notes.TypeId
WHERE WHERE
HasBeenOutput = 0 and type = 'reblog' HasBeenOutput = 0 and NoteTypes.Type = 'reblog'
order by order by
Notes.Type desc, NoteTypes.Type desc,
DateAdded desc DateAdded desc
LIMIT 100;</sql><sql name="SQL 7">WITH ReplyCounts AS ( LIMIT 100;</sql><sql name="SQL 7">WITH ReplyCounts AS (
SELECT SELECT
NoteBlogName, NoteBlogId,
COUNT(DISTINCT replyText) AS DistinctReplyCount COUNT(DISTINCT replyText) AS DistinctReplyCount
FROM Notes FROM Notes
where replyText &lt;&gt; '.' where replyText &lt;&gt; '.'
GROUP BY NoteBlogName GROUP BY NoteBlogId
) )
SELECT SELECT
n.RootBlogName || '.tumblr.com/post/' || n.PostID AS PostURL, postid, rbn.BlogName || '.tumblr.com/post/' || n.PostID AS PostURL, postid,
n.NoteBlogName, nbn.BlogName AS NoteBlogName,
n.replyText, n.replyText,
c.DistinctReplyCount c.DistinctReplyCount
FROM Notes n FROM Notes n
JOIN ReplyCounts c ON n.NoteBlogName = c.NoteBlogName JOIN ReplyCounts c ON n.NoteBlogId = c.NoteBlogId
where replyText &lt;&gt; '.' and type &lt;&gt; 'reply' JOIN BlogNames rbn ON rbn.BlogId = n.RootBlogId
--AND N.NoteBlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592', JOIN BlogNames nbn ON nbn.BlogId = n.NoteBlogId
JOIN NoteTypes t ON t.TypeId = n.TypeId
where replyText &lt;&gt; '.' and t.Type &lt;&gt; 'reply'
--AND nbn.BlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592',
--'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' ) --'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' )
order by c.DistinctReplyCount desc, n.NoteBlogName, n.DateModified desc, replyText, RootBlogName, PostID</sql><sql name="Collect">WITH PostsWithCount AS ( SELECT P.BlogName, P.PostID, 1925013599 AS LatestNoteTimestamp, P.NotesGatheredDateTime, COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT, P.HasNotesGathered, P.NotFound, P.PostDate FROM Posts P WHERE COALESCE(P.IsActive, 1) = 1 ), Unioned AS ( SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE NotFound = 0 AND HasNotesGathered = 0 UNION SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE BlogName = 'zomb-eh' AND NotFound = 0 AND NotesGatheredDateTime &lt; unixepoch('now', 'localtime', '-3 days') ) SELECT U.BlogName, U.PostID, U.LatestNoteTimestamp, U.NotesGatheredDateTime, U.CNT FROM Unioned U WHERE (U.NotesGatheredDateTime &lt; 1786134037 OR U.NotesGatheredDateTime IS NULL) ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;</sql><sql name="Del Posts">delete from posts where postid in order by c.DistinctReplyCount desc, nbn.BlogName, n.DateModified desc, replyText, rbn.BlogName, PostID</sql><sql name="Collect">WITH PostsWithCount AS ( SELECT P.BlogName, P.PostID, 1925013599 AS LatestNoteTimestamp, P.NotesGatheredDateTime, COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT, P.HasNotesGathered, P.NotFound, P.PostDate FROM Posts P WHERE COALESCE(P.IsActive, 1) = 1 ), Unioned AS ( SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE NotFound = 0 AND HasNotesGathered = 0 UNION SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE BlogName = 'zomb-eh' AND NotFound = 0 AND NotesGatheredDateTime &lt; unixepoch('now', 'localtime', '-3 days') ) SELECT U.BlogName, U.PostID, U.LatestNoteTimestamp, U.NotesGatheredDateTime, U.CNT FROM Unioned U WHERE (U.NotesGatheredDateTime &lt; 1786134037 OR U.NotesGatheredDateTime IS NULL) ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;</sql><sql name="Del Posts">delete from posts where postid in
( (
'741662499571728384', '741662499571728384',
178892849664, 178892849664,
+12 -6
View File
@@ -9,16 +9,22 @@ WHERE
ORDER BY ORDER BY
postdate desc</sql><sql name="SQL 2*">SELECT postdate desc</sql><sql name="SQL 2*">SELECT
datetime(TimeStamp, 'unixepoch'), datetime(TimeStamp, 'unixepoch'),
RootBlogName || '.tumblr.com/post/' || N.postid, rbn.BlogName || '.tumblr.com/post/' || N.postid,
*, *,
NoteBlogName || '.tumblr.com' nbn.BlogName || '.tumblr.com'
FROM FROM
Notes N Notes N
inner JOIN inner JOIN
Posts P on P.PostID = N.PostID and P.BlogName = N.RootBlogName BlogNames rbn on rbn.BlogId = N.RootBlogId
WHERE RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')␍ inner JOIN
and type like 'r%'␍ BlogNames nbn on nbn.BlogId = N.NoteBlogId
and RootBlogName = 'zomb-eh'␍ inner JOIN
NoteTypes t on t.TypeId = N.TypeId
inner JOIN
Posts P on P.PostID = N.PostID and P.BlogName = rbn.BlogName
WHERE rbn.BlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')
and t.Type like 'r%'
and rbn.BlogName = 'zomb-eh'
and P.HasImage = 1 and P.HasImage = 1
ORDER BY ORDER BY
TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project> TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project>
+6
View File
@@ -148,6 +148,12 @@ namespace URLNotesGrabberCORE
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
{ {
int postsAdded = 0; int postsAdded = 0;
// This is the mode that actually gets run day to day, so the schema migration and
// the PostType backfill have to happen here too. They used to hang off --ingest,
// --output and friends only, which meant the untyped rows this traversal creates
// could sit unrepaired indefinitely while the one command everyone runs skipped
// the fix entirely. Idempotent, so paying it on every run costs nothing.
DataAccess.EnsureTTFileHelperColumnsExist();
try try
{ {
DataAccess.EnableImportModePragmas(); DataAccess.EnableImportModePragmas();