diff --git a/AGENTS.md b/AGENTS.md index 7c47810..3143484 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,26 @@ say nothing about the item being fetched, so they must not be recorded as per-it - Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or skipped items), so a caller can distinguish that from a clean run +### `IsActive` Is Not Ours To Write +`Blogs.IsActive`, `Posts.IsActive` and `Notes.IsActive` are removal flags set by other tools +(Rolodex). `0` means removed; anything else, including `NULL`, means live. Full detail in +`URLNotesGrabberCORE/TL.db.md`. + +- **Never write any `IsActive` column.** Not in an `INSERT` column list, not in an `UPDATE`, + and never via `INSERT OR REPLACE` on these tables — that resets the column default and + un-removes the row. Re-crawling a removed row must refresh its content and leave the flag + where it was +- **Filter at selection, not at write.** Every query that *selects* posts, notes or blogs + excludes removed rows. Update statements stay keyed on a row the caller already selected; + filtering them would spend API quota and then fail to persist the result +- `Posts.IsActive` and `Notes.IsActive` are **optional** — they are absent from databases + that predate them, and naming a missing column is a hard SQLite error. Compose the filter + with `AndIsActive`/`WhereIsActive` in `DataAccess.cs`, which return + `COALESCE(IsActive, 1) = 1` only when `HasIsActiveColumn` finds the column. `Blogs.IsActive` + is not optional and is filtered directly +- Do not add these columns from this app, and do not add them to the missing-column list in + `verify-db-schema.sql` + ### Testing - No existing test suite; use xUnit if adding tests - Test critical logic: `ApiKeyPool` init, color parsing, config persistence diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 9c479bf..04235bc 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -124,6 +124,91 @@ namespace URLNotesGrabberCORE return _cachedDbPath; } + #region IsActive + + // Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does: + // 0 = removed elsewhere (Rolodex), anything else (including NULL) = live. + // + // This crawler is a reader of all three. It never writes any IsActive column -- + // no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it -- + // so a row removed in Rolodex is never resurrected by a re-crawl. + // + // Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added + // from outside this app and are absent from databases that predate them. Naming a + // missing column is a hard SQLite error ("no such column"), so every read asks the + // schema first and simply drops the filter when the column is not there. The answer + // is cached per database path, so adding the columns to a live database takes effect + // on the next run. + private static readonly Dictionary _isActiveColumnCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private static readonly object _isActiveColumnLock = new object(); + + private static bool HasIsActiveColumn(string table, string? DBPath) + { + DBPath ??= GetDefaultDbPath(); + string cacheKey = DBPath + "|" + table; + + lock (_isActiveColumnLock) + { + if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached)) + return cached; + } + + bool exists = false; + try + { + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + + using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection); + using SQLiteDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase)) + { + exists = true; + break; + } + } + } + catch (Exception ex) + { + // Breakpoint here + // An unreadable schema is treated as "no column" so the caller's query still runs. + Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}"); + } + + lock (_isActiveColumnLock) + { + _isActiveColumnCache[cacheKey] = exists; + } + + return exists; + } + + /// + /// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it + /// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive. + /// + private static string AndIsActive(string table, string alias = "", string? DBPath = null) + { + if (!HasIsActiveColumn(table, DBPath)) return string.Empty; + + string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + "."; + return $" AND COALESCE({qualifier}IsActive, 1) = 1"; + } + + /// + /// Same filter as , for a query that has no WHERE clause yet. + /// + private static string WhereIsActive(string table, string alias = "", string? DBPath = null) + { + string clause = AndIsActive(table, alias, DBPath); + return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length); + } + + #endregion IsActive + public static string Q(string input) { return "'" + input.Replace("'", "''") + "'"; @@ -464,6 +549,8 @@ namespace URLNotesGrabberCORE { if (ownsConnection) connection.Open(); + // IsActive is deliberately absent from this column list: a post removed + // elsewhere must stay removed, so the crawler never writes that flag. string sql = @"INSERT INTO Posts ( BlogName, PostID, @@ -616,6 +703,8 @@ namespace URLNotesGrabberCORE { connection2.Open(); + // INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note + // that was removed elsewhere leaves the existing row -- and its flag -- alone. string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection2)) { @@ -713,7 +802,7 @@ namespace URLNotesGrabberCORE " P.HasNotesGathered," + Environment.NewLine + " P.NotFound," + Environment.NewLine + " P.PostDate" + Environment.NewLine + - " FROM Posts P" + Environment.NewLine + + " FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine + ")," + Environment.NewLine + "Unioned AS" + Environment.NewLine + "(" + Environment.NewLine + @@ -765,8 +854,8 @@ namespace URLNotesGrabberCORE " LEFT OUTER JOIN " + Environment.NewLine + " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + " LEFT OUTER JOIN " + Environment.NewLine + - " ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " + - "WHERE NotFound = 0 " + Environment.NewLine; + " ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " + + "WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine; if (beforeDate.HasValue) { @@ -875,7 +964,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID"; + string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -918,8 +1007,8 @@ namespace URLNotesGrabberCORE string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID, MAX(Notes.timestamp) as LatestTimestamp FROM Notes - WHERE Notes.type = 'reply' - AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.') + WHERE Notes.type = 'reply' + AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @" GROUP BY Notes.RootBlogName, Notes.PostID ORDER BY LatestTimestamp ASC LIMIT @limit"; @@ -966,8 +1055,8 @@ namespace URLNotesGrabberCORE FROM Posts P INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName WHERE P.NotFound = 0 - AND N.type = 'reply' - AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.') + AND N.type = 'reply' + AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @" GROUP BY P.BlogName, P.PostID ORDER BY LatestTimestamp ASC"; @@ -1086,7 +1175,7 @@ namespace URLNotesGrabberCORE INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName - AND B.IsActive = 1 + AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @" GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);"; } @@ -1101,7 +1190,7 @@ namespace URLNotesGrabberCORE INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName - AND B.IsActive = 1 + AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @" AND ( B.LikesPulled = 0 OR COALESCE(B.LikesLastRefreshed, 0) @@ -1150,9 +1239,9 @@ namespace URLNotesGrabberCORE connection.Open(); string sql = ""; if (reblogsOnly) - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; else - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -1191,9 +1280,9 @@ namespace URLNotesGrabberCORE connection.Open(); string sql = ""; if (reblogsOnly) - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; else - sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; + sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -1225,7 +1314,7 @@ namespace URLNotesGrabberCORE DBPath ??= GetDefaultDbPath(); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); - string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; + string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath); using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) @@ -1895,6 +1984,8 @@ namespace URLNotesGrabberCORE { if (ownsConnection) connection.Open(); + // As in AddPost, IsActive is never written -- neither here nor in the + // UPDATE below, which is why an ingest cannot un-remove a post. string insertSql = @"INSERT INTO Posts ( BlogName, PostID, reblogURL, PostDate, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, @@ -2025,7 +2116,7 @@ namespace URLNotesGrabberCORE PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, PostType, HasImage, DateCreated, DateModified - FROM Posts WHERE BlogName = @BlogName"; + FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath); using var cmd = new SQLiteCommand(sql, connection); cmd.Parameters.AddWithValue("@BlogName", blogName); using var reader = cmd.ExecuteReader(); @@ -2074,7 +2165,7 @@ namespace URLNotesGrabberCORE PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, PostType, HasImage, DateCreated, DateModified - FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID"; + FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath); using var cmd = new SQLiteCommand(sql, connection); cmd.Parameters.AddWithValue("@BlogName", blogName); cmd.Parameters.AddWithValue("@PostID", postId); @@ -2124,7 +2215,7 @@ namespace URLNotesGrabberCORE PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, PostType, HasImage, DateCreated, DateModified - FROM Posts WHERE PostID = @PostID LIMIT 1"; + FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1"; using var cmd = new SQLiteCommand(sql, connection); cmd.Parameters.AddWithValue("@PostID", postId); using var reader = cmd.ExecuteReader(); diff --git a/URLNotesGrabberCORE/TL.db.md b/URLNotesGrabberCORE/TL.db.md index 83cd2ef..8fed093 100644 --- a/URLNotesGrabberCORE/TL.db.md +++ b/URLNotesGrabberCORE/TL.db.md @@ -262,6 +262,47 @@ under its Posts and Notes pages. Removing a blog hides the blog, not what it col --- +## `Posts.IsActive` and `Notes.IsActive` — optional, and not in this database yet + +The same flag is being extended to the two content tables, with the same meaning: `0` is +removed, anything else — including `NULL` — is live. **Neither column exists in the live +`TL.db` as of 2026-07-29**; the DDL quoted above for `Posts` and `Notes` is complete. Like +`Blogs.IsActive`, they are written from outside this crawler. + +The crawler therefore treats both as optional, and as nothing it owns: + +- **It never writes them.** No `INSERT` column list names `IsActive`, no `UPDATE` sets it, + and `MapPrefixToColumn` — the only place a column name is chosen at runtime — cannot map + to it. Re-crawling a removed post or note refreshes its content and leaves the flag at + `0`. There is no `INSERT OR REPLACE` on `Posts` or `Notes` for a default to be reset by. +- **It filters on them only when they exist.** `HasIsActiveColumn` in `DataAccess.cs` asks + `PRAGMA table_info` once per table per database path and caches the answer; the filter + is `COALESCE(IsActive, 1) = 1`, and it is dropped entirely when the column is absent. + Naming a missing column is a hard SQLite error, so this is what lets one build run + against databases on both sides of the change. The cache lives for the process — adding + the columns to a live database takes effect on the next run. + +Every read that selects posts or notes carries the filter: `GetPosts`, `GetReplies`, +`GetRepliesWithMissingText`, `GetRepliesWithFilledText`, `GetAllPostTextColumns`, +`GetAllPostsForBlog`, `GetPost`, `GetPostByIdAnyBlog`, and the engagement queries that +count or join `Notes` (`GetBlogs`, `GetBlogsAll`, `GetBlogsForLikes`). The one deliberate +omission is the `LEFT JOIN Notes` in `GetPosts`: nothing is selected from it and it can +neither add nor remove a row, so filtering it would buy nothing. + +`LegacyPostsDbImporter` is unfiltered too — it reads a foreign legacy database whose +`Posts` table is not this schema. + +Two consequences worth stating plainly, both inherited from how `Blogs.IsActive` is +handled: + +1. **Removal hides a row; it does not freeze it.** The write paths are keyed on a post the + caller already selected, so an ingest or a correction run still overwrites the content + of a removed post. Only selection is filtered. +2. **`NULL` is live.** Write `0` or `1`, not `NULL`, but a `NULL` leaves the row visible + rather than stranding it. + +--- + ## Reproducing the numbers ```sql diff --git a/verify-db-schema.sql b/verify-db-schema.sql index 194d009..f4a25a7 100644 --- a/verify-db-schema.sql +++ b/verify-db-schema.sql @@ -142,6 +142,9 @@ ORDER BY et.tbl; -- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected -- list above. Informational only -- e.g. a NEWER backup, or a column this -- script's expected-list hasn't been updated for. Not an error by itself. +-- Posts.IsActive and Notes.IsActive are listed here and NOT in 1a on +-- purpose: they are written by other tools, the app only reads them when +-- present, and it must not be told to add them. See TL.db.md. WITH expected(tbl, col) AS ( VALUES ('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'), @@ -150,14 +153,14 @@ WITH expected(tbl, col) AS ( ('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'), ('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'), ('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'), - ('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'), + ('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),('Posts','IsActive'), ('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'), ('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'), ('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'), ('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'), ('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'), ('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'), - ('Notes','replyText'), + ('Notes','replyText'),('Notes','IsActive'), ('DailyAPICount','Date'),('DailyAPICount','APICount'), ('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'), ('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')