diff --git a/AGENTS.md b/AGENTS.md index b602bda..066e933 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,36 @@ say nothing about the item being fetched, so they must not be recorded as per-it - Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or skipped items), so a caller can distinguish that from a clean run +### `Notes` Stores Integer IDs, Not Names +As of 2026-08-07 `Notes.RootBlogName`, `NoteBlogName` and `Type` are gone, replaced by +`RootBlogId`, `NoteBlogId` and `TypeId` resolving through the `BlogNames` and `NoteTypes` +lookup tables. There is no compatibility view — naming an old column is a hard SQLite +error, so unlike `IsActive` this is a hard cut with no runtime probe. Full detail in +`URLNotesGrabberCORE/TL.db.md`. + +- **Joining `Notes` to `Blogs` goes through `Blogs.BlogId`**, not `BlogNames`: + `FROM Blogs B INNER JOIN Notes N ON N.NoteBlogId = B.BlogId`. Routing it through + `BlogNames` adds a hop and ends in the text comparison the migration removed +- **Joining `Notes` to `Posts` is the opposite** — `Posts` has only `BlogName`, so it must + go through `BlogNames` (`GetRepliesWithFilledText`). This is the only such join +- **Resolve a name by filtering the lookup, never by scanning `Notes`**: + `WHERE NoteBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @name)`. The subquery + is a unique-index probe on 20k rows and does not show against the 1.18M-row table +- **`AddNote` registers both blog names *and* the note type** with `INSERT OR IGNORE` + before inserting, all in one transaction. `NoteTypes` is a table rather than a `CHECK` + constraint precisely so an unseen type is an `INSERT`; without that registration it + would resolve to `NULL` and fail the `NOT NULL` on `TypeId`, losing the note +- **`Blogs.BlogId` is NULL on 168,202 of 188,620 rows** — every blog that has never + appeared in a note. An inner join on it silently drops them. Correct for engagement + queries, wrong for anything listing the registry +- **IDs are stable and must never be renumbered.** They are stored in 1.18M `Notes` rows. + A blog renamed upstream gets a new `BlogNames` row, not an edited one +- Prefer `TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply')` over a hardcoded + ID. A negated `TypeId NOT IN (SELECT …)` is only correct because `TypeId` is `NOT NULL` +- Duplicate-key detection uses `IsNotesDuplicateKey`, which matches the constraint and the + table rather than an exact column list. The old literal string comparison broke silently + on this rename — do not reintroduce one + ### `IsActive` Is Not Ours To Write `Blogs.IsActive`, `Posts.IsActive` and `Notes.IsActive` are removal flags set by other tools (Rolodex). `0` means removed; anything else, including `NULL`, means live. Full detail in diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index b2802f0..dd7fd1e 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -213,6 +213,59 @@ namespace URLNotesGrabberCORE #endregion IsActive + #region Notes integer schema + + // Notes stopped storing names on 2026-08-07: RootBlogName/NoteBlogName/Type became + // RootBlogId/NoteBlogId/TypeId, resolved through BlogNames and NoteTypes. There is no + // compatibility view -- a query naming an old column fails outright, so this is a hard + // cut rather than an optional column like IsActive. See TL.db.md. + // + // Two shapes recur below and are spelled out inline rather than hidden behind a helper, + // so that every statement reads as the SQL it actually runs: + // (SELECT BlogId FROM BlogNames WHERE BlogName = @name) -- unique-index probe, 20k rows + // (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') -- 5 rows, effectively free + // Joining Notes to Blogs is the one case that must NOT route through BlogNames: Blogs + // carries its own BlogId, so N.NoteBlogId = B.BlogId is a single integer hop. Joining + // Notes to Posts is the opposite case -- Posts has only BlogName, so it has to go + // through BlogNames. + + /// + /// True when the exception is a duplicate-key collision on Notes. The message embeds the + /// primary key's column names, which the integer migration renamed, so this matches on the + /// constraint and the table instead of on an exact column list -- a literal comparison + /// silently inverts into "log every error" the next time a column is renamed. + /// + private static bool IsNotesDuplicateKey(Exception ex) + { + return ex.Message.Contains("UNIQUE constraint failed", StringComparison.OrdinalIgnoreCase) + && ex.Message.Contains("Notes.", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Gives a blog name an ID if it does not have one. No read-back and no round trip -- a + /// name that is already registered keeps the ID that 1.18M Notes rows point at. + /// + private static void RegisterBlogName(SQLiteConnection connection, SQLiteTransaction? transaction, string blogName) + { + using SQLiteCommand command = new SQLiteCommand("INSERT OR IGNORE INTO BlogNames (BlogName) VALUES (@BlogName)", connection, transaction); + command.Parameters.AddWithValue("@BlogName", blogName); + command.ExecuteNonQuery(); + } + + /// + /// Same, for a note type. NoteTypes is a table rather than a CHECK constraint precisely so + /// that a type this crawler has not seen before is an INSERT and not a schema migration -- + /// without this the type would resolve to NULL and fail the NOT NULL on Notes.TypeId. + /// + private static void RegisterNoteType(SQLiteConnection connection, SQLiteTransaction? transaction, string type) + { + using SQLiteCommand command = new SQLiteCommand("INSERT OR IGNORE INTO NoteTypes (Type) VALUES (@Type)", connection, transaction); + command.Parameters.AddWithValue("@Type", type); + command.ExecuteNonQuery(); + } + + #endregion Notes integer schema + public static string Q(string input) { return "'" + input.Replace("'", "''") + "'"; @@ -371,7 +424,10 @@ namespace URLNotesGrabberCORE connection.Close(); connection.Open(); - string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';"; + // No column default: the migrated schema dropped the DEFAULT '.' that + // is how 1.1M rows acquired a placeholder nobody wrote. New rows get + // NULL, which every reader here already treats as "no reply text". + string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT;"; using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection)) { addCommand.ExecuteNonQuery(); @@ -703,63 +759,85 @@ namespace URLNotesGrabberCORE try { AddBlog(noteBlogName, false, DBPath); } catch { } using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); + int rowsInserted = 0; try { connection2.Open(); - // INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note - // that was removed elsewhere leaves the existing row -- and its flag -- alone. - string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)"; - using (SQLiteCommand command = new SQLiteCommand(sql, connection2)) + // Notes stores integer IDs, so both participants and the type have to exist in + // their lookup table before the note can point at them. + // + // All four statements run in one transaction so a crash cannot leave a name or a + // type registered with no note. The transaction is committed before the console + // output below, which sleeps -- a write lock must not be held across that. + using (SQLiteTransaction transaction = connection2.BeginTransaction()) { - command.Parameters.AddWithValue("@rootBlogName", rootBlogName); - command.Parameters.AddWithValue("@noteBlogName", noteBlogName); - command.Parameters.AddWithValue("@PostID", postID); - command.Parameters.AddWithValue("@TimeStamp", timestamp); - command.Parameters.AddWithValue("@Type", type ?? string.Empty); - command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); - command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); - command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + RegisterBlogName(connection2, transaction, rootBlogName); + RegisterBlogName(connection2, transaction, noteBlogName); + RegisterNoteType(connection2, transaction, type ?? string.Empty); - int rowsInserted = command.ExecuteNonQuery(); + // INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note + // that was removed elsewhere leaves the existing row -- and its flag -- alone. + string sql = "INSERT OR IGNORE INTO Notes (RootBlogId, NoteBlogId, PostID, TimeStamp, TypeId, DatetimeCrawled, DateModified, DateCreated) " + + "SELECT (SELECT BlogId FROM BlogNames WHERE BlogName = @rootBlogName), " + + " (SELECT BlogId FROM BlogNames WHERE BlogName = @noteBlogName), " + + " @PostID, @TimeStamp, " + + " (SELECT TypeId FROM NoteTypes WHERE Type = @Type), " + + " @DatetimeCrawled, @DateModified, @DateCreated"; - if (rowsInserted == 1) + using (SQLiteCommand command = new SQLiteCommand(sql, connection2, transaction)) { - ConsoleColor previousColor = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Green; - Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); - Console.ForegroundColor = previousColor; - Thread.Sleep(250); // Brief pause to make new notes more noticeable in the console output - } - else - { - Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); + command.Parameters.AddWithValue("@rootBlogName", rootBlogName); + command.Parameters.AddWithValue("@noteBlogName", noteBlogName); + command.Parameters.AddWithValue("@PostID", postID); + command.Parameters.AddWithValue("@TimeStamp", timestamp); + command.Parameters.AddWithValue("@Type", type ?? string.Empty); + command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + + rowsInserted = command.ExecuteNonQuery(); } - // Only update HasBeenOutput if a new note was inserted - if (rowsInserted == 1) + transaction.Commit(); + } + + if (rowsInserted == 1) + { + ConsoleColor previousColor = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); + Console.ForegroundColor = previousColor; + Thread.Sleep(250); // Brief pause to make new notes more noticeable in the console output + } + else + { + Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); + } + + // Only update HasBeenOutput if a new note was inserted + if (rowsInserted == 1) + { + try { - try + // HasBeenOutput IS NULL still counts as a change: the selection queries + // test HasBeenOutput = 0, which a NULL would never match. + string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)"; + using (var updateCommand = new SQLiteCommand(updateSql, connection2)) { - // HasBeenOutput IS NULL still counts as a change: the selection queries - // test HasBeenOutput = 0, which a NULL would never match. - string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)"; - using (var updateCommand = new SQLiteCommand(updateSql, connection2)) - { - updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName); - updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); - updateCommand.ExecuteNonQuery(); - } + updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName); + updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + updateCommand.ExecuteNonQuery(); } - catch { } } + catch { } } } catch (Exception ex) { // Breakpoint here - if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") + if (!IsNotesDuplicateKey(ex)) { Console.WriteLine(ex.Message); Console.WriteLine("^^^^^ - SHORTCUT"); @@ -850,6 +928,10 @@ namespace URLNotesGrabberCORE } else { + // The LEFT OUTER JOIN to Notes that used to sit here has been dropped rather + // than ported. Nothing was selected from it, a LEFT JOIN cannot remove a row, + // and the GROUP BY below collapsed the rows it duplicated -- so it could not + // affect the result, and it cost a join against 1.18M rows on every pass. sql = "SELECT " + " MAX(Posts.BlogName) as BlogName, " + Environment.NewLine + " Posts.PostID, " + Environment.NewLine + @@ -859,8 +941,6 @@ namespace URLNotesGrabberCORE "FROM " + Environment.NewLine + " Posts " + Environment.NewLine + " LEFT OUTER JOIN " + Environment.NewLine + - " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + - " LEFT OUTER JOIN " + Environment.NewLine + " ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " + "WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine; @@ -971,7 +1051,11 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID"; + string sql = "SELECT DISTINCT BN.BlogName as blogName, N.PostID" + + " FROM Notes N" + + " INNER JOIN BlogNames BN ON BN.BlogId = N.RootBlogId" + + " WHERE N.TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply')" + AndIsActive("Notes", "N", DBPath) + + " ORDER BY BN.BlogName, N.PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -1011,12 +1095,15 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID, - MAX(Notes.timestamp) as LatestTimestamp - FROM Notes - WHERE Notes.type = 'reply' - AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @" - GROUP BY Notes.RootBlogName, Notes.PostID + // Grouped on the integer rather than the name: the group key is what gets sorted, + // and BN.BlogName comes along for free off the join. + string sql = @"SELECT BN.BlogName as blogName, N.PostID, + MAX(N.TimeStamp) as LatestTimestamp + FROM Notes N + INNER JOIN BlogNames BN ON BN.BlogId = N.RootBlogId + WHERE N.TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') + AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Notes", "N", DBPath) + @" + GROUP BY N.RootBlogId, N.PostID ORDER BY LatestTimestamp ASC LIMIT @limit"; @@ -1058,11 +1145,15 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = @"SELECT DISTINCT P.BlogName, P.PostID, MAX(N.timestamp) as LatestTimestamp + // Posts carries only BlogName, so this is the one join to Notes that has to go + // through BlogNames -- there is no Posts.BlogId to hop on. The name predicate is + // pushed into the 20k-row lookup, which then feeds integers to the Notes key. + string sql = @"SELECT DISTINCT P.BlogName, P.PostID, MAX(N.TimeStamp) as LatestTimestamp FROM Posts P - INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName + INNER JOIN BlogNames RBN ON RBN.BlogName = P.BlogName + INNER JOIN Notes N ON N.RootBlogId = RBN.BlogId AND N.PostID = P.PostID WHERE P.NotFound = 0 - AND N.type = 'reply' + AND N.TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @" GROUP BY P.BlogName, P.PostID ORDER BY LatestTimestamp ASC"; @@ -1160,6 +1251,11 @@ namespace URLNotesGrabberCORE { connection.Open(); string sql; + + // The two Notes branches below join on Blogs.BlogId, which is NULL for the 168k + // registry rows that have never appeared in a note. The inner join drops them, + // which is correct here -- both branches already require a note to exist -- but + // it is the wrong shape for anything that lists the registry. if (!string.IsNullOrEmpty(specificBlog)) { // Specific blog: always process, bypass cooldown @@ -1179,12 +1275,12 @@ namespace URLNotesGrabberCORE COALESCE(B.LikesCursor, 0), COALESCE(B.LikesNewestTimestamp, 0) FROM Blogs B - INNER JOIN Notes N ON N.NoteBlogName = B.BlogName + INNER JOIN Notes N ON N.NoteBlogId = B.BlogId WHERE N.TimeStamp >= 1535778000 - AND N.rootBlogName = B.BlogName + AND N.RootBlogId = B.BlogId AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @" GROUP BY B.BlogName - ORDER BY MIN(N.Timestamp);"; + ORDER BY MIN(N.TimeStamp);"; } else { @@ -1194,9 +1290,9 @@ namespace URLNotesGrabberCORE COALESCE(B.LikesCursor, 0), COALESCE(B.LikesNewestTimestamp, 0) FROM Blogs B - INNER JOIN Notes N ON N.NoteBlogName = B.BlogName + INNER JOIN Notes N ON N.NoteBlogId = B.BlogId WHERE N.TimeStamp >= 1535778000 - AND N.rootBlogName = B.BlogName + AND N.RootBlogId = B.BlogId AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @" AND ( B.LikesPulled = 0 @@ -1204,7 +1300,7 @@ namespace URLNotesGrabberCORE < (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400)) ) GROUP BY B.BlogName - ORDER BY MIN(N.Timestamp);"; + ORDER BY MIN(N.TimeStamp);"; } using (SQLiteCommand command = new SQLiteCommand(sql, connection)) @@ -1244,11 +1340,14 @@ namespace URLNotesGrabberCORE try { connection.Open(); + // Blogs is reached in one integer hop off Blogs.BlogId, not through BlogNames -- + // that would add a hop and end in the text comparison the migration removed. + // The negated form is only correct because Notes.TypeId is NOT NULL. string sql = ""; if (reblogsOnly) - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND N.TypeId IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply', 'posted')) AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top"; else - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND N.TypeId NOT IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply', 'posted')) AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -1287,9 +1386,9 @@ namespace URLNotesGrabberCORE connection.Open(); string sql = ""; if (reblogsOnly) - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND N.TypeId IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply', 'posted')) AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top"; else - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -1528,7 +1627,10 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID AND IFNULL(timestamp, 0) <> @timestamp"; + string sql = "UPDATE Notes SET TimeStamp = @timestamp, DateModified = @dateModified " + + "WHERE RootBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @rootBlogName) " + + "AND NoteBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @noteBlogName) " + + "AND PostID = @postID AND IFNULL(TimeStamp, 0) <> @timestamp"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@timestamp", timestamp); @@ -1542,7 +1644,7 @@ namespace URLNotesGrabberCORE catch (Exception ex) { // Breakpoint here - if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") + if (!IsNotesDuplicateKey(ex)) { Console.WriteLine(ex.Message); Console.WriteLine("^^^^^ - SHORTCUT"); @@ -1828,10 +1930,15 @@ namespace URLNotesGrabberCORE { connection.Open(); - //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; // Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now. // Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text. - string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.') AND (replyText IS NULL OR replyText <> @replyText)"; + // The ABS() term cannot use an index on TimeStamp, before or after the integer schema; the NoteBlogId probe is what keeps this off a full scan. + string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified " + + "WHERE NoteBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @noteBlogName) " + + "AND ABS(TimeStamp - @TimeStamp) <= 5 " + + "AND TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') " + + "AND (replyText IS NULL OR replyText = '' OR replyText = '.') " + + "AND (replyText IS NULL OR replyText <> @replyText)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@replyText", replyText ?? "?"); @@ -1872,7 +1979,11 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText"; + string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified " + + "WHERE RootBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @rootBlogName) " + + "AND PostID = @PostID " + + "AND TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') " + + "AND IFNULL(replyText, '.') <> @replyText"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@replyText", replyText ?? "."); diff --git a/URLNotesGrabberCORE/TL.db.md b/URLNotesGrabberCORE/TL.db.md index 3161eb8..a4169cb 100644 --- a/URLNotesGrabberCORE/TL.db.md +++ b/URLNotesGrabberCORE/TL.db.md @@ -313,9 +313,17 @@ code to this table's contents, so prefer the join in anything long-lived. ## Porting to the integer schema -Everything here was checked against the live 148 MB file. There are roughly 14 affected -call sites in `DataAccess.cs` and 16 in `RolodexRepository.cs`. TumblThree needs no -changes — its single statement touches `Blogs.IsActive` and `BlogName` only. +Everything here was checked against the live 148 MB file. There were 14 affected call +sites in `DataAccess.cs` and 16 in `RolodexRepository.cs`. TumblThree needs no changes — +its single statement touches `Blogs.IsActive` and `BlogName` only. + +**`DataAccess.cs` is ported.** All 14 sites now read the integer schema, `AddNote` +registers names and types before inserting, and `verify-db-schema.sql` reports a +pre-migration file rather than letting the app fail on it. `RolodexRepository.cs` lives in +the [Rolodex](https://git.basso.land/jim/Rolodex) repository and is not covered by that +work. One site was dropped rather than translated: the `LEFT JOIN Notes` in `GetPosts` +selected nothing and was collapsed by the query's own `GROUP BY`, so it could not affect +the result. ### Column mapping @@ -426,7 +434,11 @@ UNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Not at two call sites to decide whether to swallow an exception. SQLite now emits the *new* column names, so those comparisons no longer match and real errors will surface where -they used to be silently ignored — or vice versa. Both sites need updating. +they used to be silently ignored — or vice versa. + +Both sites now go through `IsNotesDuplicateKey` in `DataAccess.cs`, which matches on +`UNIQUE constraint failed` plus `Notes.` rather than on the column list. A literal +comparison is what broke here; the next rename should not break it again. ### Updating notes @@ -526,7 +538,7 @@ Crawler bookkeeping. Rolodex ignores all of these. `DataAccess.cs` joins on it to decide what to collect: ```sql --- as it will read after the integer-schema port; see the porting guide above +-- shape only; the ported GetBlogs joins Blogs directly on BlogId and needs no BlogNames hop SELECT bn.BlogName, count(*) FROM Notes n JOIN Blogs b ON b.BlogId = n.NoteBlogId diff --git a/verify-db-schema.sql b/verify-db-schema.sql index f4a25a7..43a4dea 100644 --- a/verify-db-schema.sql +++ b/verify-db-schema.sql @@ -79,18 +79,35 @@ WITH expected(tbl, col, alter_stmt) AS ( ('Blogs','LikesLastRefreshed', 'ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;'), ('Blogs','LikesLastNewCount', 'ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;'), ('Blogs','TTFolderPath', 'ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;'), + -- Blogs.BlogId (2026-08-07) is the single-hop join key into Notes. Deliberately NOT + -- auto-fixable: an added-but-empty BlogId makes every engagement join return zero + -- rows silently, which is worse than the hard error a missing column gives. + ('Blogs','BlogId', 'MANUAL REVIEW - see query 1d: run normalize-notes.sql'), -- Notes (base columns: manual review if missing) - ('Notes','RootBlogName', 'MANUAL REVIEW - base/PK column missing'), + -- Integer IDs since 2026-08-07. RootBlogName/NoteBlogName/Type are GONE, not renamed + -- in place -- a backup that still has them needs normalize-notes.sql, not an ALTER. + -- Query 1d below reports exactly that case. + ('Notes','RootBlogId', 'MANUAL REVIEW - see query 1d: pre-2026-08-07 name schema, or damaged'), ('Notes','PostID', 'MANUAL REVIEW - base/PK column missing'), - ('Notes','NoteBlogName', 'MANUAL REVIEW - base/PK column missing'), + ('Notes','NoteBlogId', 'MANUAL REVIEW - see query 1d: pre-2026-08-07 name schema, or damaged'), ('Notes','TimeStamp', 'MANUAL REVIEW - base/PK column missing'), - ('Notes','Type', 'MANUAL REVIEW - base/PK column missing'), + ('Notes','TypeId', 'MANUAL REVIEW - see query 1d: pre-2026-08-07 name schema, or damaged'), ('Notes','DatetimeCrawled', 'MANUAL REVIEW - base column missing'), ('Notes','DateModified', 'MANUAL REVIEW - base column missing'), ('Notes','DateCreated', 'MANUAL REVIEW - base column missing'), -- Notes (additive migration column, auto-fixable) - ('Notes','replyText', 'ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT ''.'';'), + -- No DEFAULT: the migrated schema dropped it, so new rows get NULL rather than a + -- placeholder. EnsureReplyTextColumnExists in DataAccess.cs adds it the same way. + ('Notes','replyText', 'ALTER TABLE Notes ADD COLUMN replyText TEXT;'), + + -- BlogNames / NoteTypes (the lookup tables Notes resolves its IDs through, 2026-08-07). + -- Not auto-fixable: an empty BlogNames does not mean "add the table", it means the + -- Notes rows have nothing to resolve against. Rebuild with normalize-notes.sql. + ('BlogNames','BlogId', 'MANUAL REVIEW - see query 1d: run normalize-notes.sql'), + ('BlogNames','BlogName', 'MANUAL REVIEW - see query 1d: run normalize-notes.sql'), + ('NoteTypes','TypeId', 'MANUAL REVIEW - see query 1d: run normalize-notes.sql'), + ('NoteTypes','Type', 'MANUAL REVIEW - see query 1d: run normalize-notes.sql'), -- DailyAPICount (base columns) ('DailyAPICount','Date', 'MANUAL REVIEW - base/PK column missing'), @@ -106,6 +123,8 @@ actual(tbl, col) AS ( SELECT 'Posts', name FROM pragma_table_info('Posts') UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs') UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes') + UNION ALL SELECT 'BlogNames', name FROM pragma_table_info('BlogNames') + UNION ALL SELECT 'NoteTypes', name FROM pragma_table_info('NoteTypes') UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount') UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState') UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta') @@ -127,7 +146,7 @@ ORDER BY (e.alter_stmt LIKE 'ALTER%') DESC, e.tbl, e.col; -- 1b. MISSING TABLES: expected tables that don't exist at all in this DB. -- Zero rows = good. WITH expected_tables(tbl) AS ( - VALUES ('Posts'),('Blogs'),('Notes'),('DailyAPICount'), + VALUES ('Posts'),('Blogs'),('Notes'),('BlogNames'),('NoteTypes'),('DailyAPICount'), ('ApiKeyPoolState'),('ApiKeyPoolMeta') ) SELECT et.tbl AS missing_table @@ -157,10 +176,12 @@ WITH expected(tbl, col) AS ( ('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'), ('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'), ('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'), - ('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'), - ('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'), - ('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'), + ('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),('Blogs','BlogId'), + ('Notes','RootBlogId'),('Notes','PostID'),('Notes','NoteBlogId'),('Notes','TimeStamp'), + ('Notes','TypeId'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'), ('Notes','replyText'),('Notes','IsActive'), + ('BlogNames','BlogId'),('BlogNames','BlogName'), + ('NoteTypes','TypeId'),('NoteTypes','Type'), ('DailyAPICount','Date'),('DailyAPICount','APICount'), ('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'), ('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex') @@ -169,6 +190,8 @@ actual(tbl, col) AS ( SELECT 'Posts', name FROM pragma_table_info('Posts') UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs') UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes') + UNION ALL SELECT 'BlogNames', name FROM pragma_table_info('BlogNames') + UNION ALL SELECT 'NoteTypes', name FROM pragma_table_info('NoteTypes') UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount') UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState') UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta') @@ -181,6 +204,25 @@ WHERE e.col IS NULL ORDER BY a.tbl, a.col; +-- 1d. PRE-MIGRATION DATABASE: a backup from before 2026-08-07, when Notes still +-- stored names. Zero rows = good. +-- +-- This is the one failure SECTION 2 cannot fix. Notes.RootBlogName / +-- NoteBlogName / Type were replaced by RootBlogId / NoteBlogId / TypeId +-- resolving through BlogNames and NoteTypes -- a data migration, not an +-- ADD COLUMN. There is no compatibility view, so the current code fails +-- outright ("no such column: RootBlogId") against such a file. +-- +-- Fix: run normalize-notes.sql against a COPY of the backup, then re-run +-- SECTION 1. Do not hand-add the ID columns: they would be empty, and an +-- empty NoteBlogId is indistinguishable from a note by blog #0. +SELECT 'Notes still stores names -- run normalize-notes.sql on a copy' AS pre_migration_schema, + group_concat(name, ', ') AS legacy_columns_found +FROM pragma_table_info('Notes') +WHERE lower(name) IN ('rootblogname','noteblogname','type') +HAVING COUNT(*) > 0; + + -- ============================================================================ -- SECTION 2 -- FIX (opt-in, additive only) -- @@ -190,6 +232,9 @@ ORDER BY a.tbl, a.col; -- "duplicate column name" error and changes nothing -- just run the flagged -- subset. These are the 8 additive migration columns and nothing else; the -- likes high-water-mark reset is intentionally NOT included. +-- +-- Nothing here addresses query 1d. The Notes integer schema is a data migration +-- (normalize-notes.sql) and cannot be reached by adding columns. -- ============================================================================ -- ALTER TABLE Posts ADD COLUMN PostType TEXT; @@ -199,4 +244,4 @@ ORDER BY a.tbl, a.col; -- ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0; -- ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0; -- ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT; --- ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.'; +-- ALTER TABLE Notes ADD COLUMN replyText TEXT;