feat: honor optional Posts.IsActive and Notes.IsActive

Both columns carry the meaning Blogs.IsActive has: 0 = removed by another
tool, anything else (including NULL) = live. Neither exists in the live
TL.db yet, and both are added from outside this crawler, so the code has
to work on databases either side of the change - naming a missing column
is a hard SQLite error.

HasIsActiveColumn asks PRAGMA table_info once per table per database path
and caches it; AndIsActive/WhereIsActive return "COALESCE(IsActive, 1) = 1"
or an empty string. Every read that selects posts or notes now carries the
filter: GetPosts (both branches, including the per-blog count subquery),
GetReplies, GetRepliesWithMissingText, GetRepliesWithFilledText,
GetAllPostTextColumns, GetAllPostsForBlog, GetPost, GetPostByIdAnyBlog,
and the engagement queries that count or join Notes - GetBlogs,
GetBlogsAll and both note-joining variants of GetBlogsForLikes.

The LEFT JOIN Notes in GetPosts is left alone on purpose: nothing is
selected from it and it can neither add nor remove a row.
LegacyPostsDbImporter is left alone too - it reads a foreign legacy
schema.

Writes were already safe and are documented rather than changed: no
INSERT column list names IsActive, no UPDATE sets it, MapPrefixToColumn
cannot map to it, and there is no INSERT OR REPLACE on Posts or Notes for
a column default to be reset by. Re-crawling a removed row refreshes its
content and leaves the flag at 0. As with Blogs, exclusion belongs at
selection, so the update paths stay keyed on rows the caller already
chose.

Verified against three synthetic databases - no IsActive columns, columns
present with a removed post and its notes, and columns present but NULL -
by running every affected reader: the queries are valid in all three, the
removed rows drop out only where the columns exist, NULL reads as live,
and AddPost/AddNote/UpsertPostFromTextFile/UpdatePostContentFields leave
an IsActive = 0 row at 0.

Co-Authored-By: Claude Opus 5 <[email protected]>

URLNotesGrabberCORE/DataAccess.cs

@@ -120,6 +120,91 @@ namespace URLNotesGrabberCORE
             return _cachedDbPath;
         }
 
+        #region IsActive
+
+        // Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does:
+        // 0 = removed elsewhere (Rolodex), anything else (including NULL) = live.
+        //
+        // This crawler is a reader of all three. It never writes any IsActive column --
+        // no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it --
+        // so a row removed in Rolodex is never resurrected by a re-crawl.
+        //
+        // Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added
+        // from outside this app and are absent from databases that predate them. Naming a
+        // missing column is a hard SQLite error ("no such column"), so every read asks the
+        // schema first and simply drops the filter when the column is not there. The answer
+        // is cached per database path, so adding the columns to a live database takes effect
+        // on the next run.
+        private static readonly Dictionary<string, bool> _isActiveColumnCache =
+            new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
+        private static readonly object _isActiveColumnLock = new object();
+
+        private static bool HasIsActiveColumn(string table, string? DBPath)
+        {
+            DBPath ??= GetDefaultDbPath();
+            string cacheKey = DBPath + "|" + table;
+
+            lock (_isActiveColumnLock)
+            {
+                if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached))
+                    return cached;
+            }
+
+            bool exists = false;
+            try
+            {
+                using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+                connection.Open();
+
+                using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection);
+                using SQLiteDataReader reader = command.ExecuteReader();
+                while (reader.Read())
+                {
+                    if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase))
+                    {
+                        exists = true;
+                        break;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                // Breakpoint here
+                // An unreadable schema is treated as "no column" so the caller's query still runs.
+                Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}");
+            }
+
+            lock (_isActiveColumnLock)
+            {
+                _isActiveColumnCache[cacheKey] = exists;
+            }
+
+            return exists;
+        }
+
+        /// <summary>
+        /// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it
+        /// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive.
+        /// </summary>
+        private static string AndIsActive(string table, string alias = "", string? DBPath = null)
+        {
+            if (!HasIsActiveColumn(table, DBPath)) return string.Empty;
+
+            string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + ".";
+            return $" AND COALESCE({qualifier}IsActive, 1) = 1";
+        }
+
+        /// <summary>
+        /// Same filter as <see cref="AndIsActive"/>, for a query that has no WHERE clause yet.
+        /// </summary>
+        private static string WhereIsActive(string table, string alias = "", string? DBPath = null)
+        {
+            string clause = AndIsActive(table, alias, DBPath);
+            return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length);
+        }
+
+        #endregion IsActive
+
         public static string Q(string input)
         {
             return "'" + input.Replace("'", "''") + "'";
@@ -460,6 +545,8 @@ namespace URLNotesGrabberCORE
             {
                 if (ownsConnection) connection.Open();
 
+                // IsActive is deliberately absent from this column list: a post removed
+                // elsewhere must stay removed, so the crawler never writes that flag.
                 string sql = @"INSERT INTO Posts (
                                 BlogName,
                                 PostID,
@@ -593,6 +680,8 @@ namespace URLNotesGrabberCORE
             {
                 connection2.Open();
 
+                // INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note
+                // that was removed elsewhere leaves the existing row -- and its flag -- alone.
                 string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
                 {
@@ -690,7 +779,7 @@ namespace URLNotesGrabberCORE
                           "        P.HasNotesGathered," + Environment.NewLine +
                           "        P.NotFound," + Environment.NewLine +
                           "        P.PostDate" + Environment.NewLine +
-                          "    FROM Posts P" + Environment.NewLine +
+                          "    FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
                           ")," + Environment.NewLine +
                           "Unioned AS" + Environment.NewLine +
                           "(" + Environment.NewLine +
@@ -742,8 +831,8 @@ namespace URLNotesGrabberCORE
                                  "      LEFT OUTER JOIN " + Environment.NewLine +
                                  "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
                                  "      LEFT OUTER JOIN " + Environment.NewLine +
-                                 "   (      select BlogName, count(PostID) as CNT from Posts group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
+                                 "   (      select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
-                                 "WHERE NotFound = 0 " + Environment.NewLine;
+                                 "WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
 
                     if (beforeDate.HasValue)
                     {
@@ -852,7 +941,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "SELECT	distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
+                string sql = "SELECT	distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -895,8 +984,8 @@ namespace URLNotesGrabberCORE
                 string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
                              MAX(Notes.timestamp) as LatestTimestamp
                              FROM Notes 
-                             WHERE Notes.type = 'reply' 
+                             WHERE Notes.type = 'reply'
-                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @"
                              GROUP BY Notes.RootBlogName, Notes.PostID
                              ORDER BY LatestTimestamp ASC
                              LIMIT @limit";
@@ -943,8 +1032,8 @@ namespace URLNotesGrabberCORE
                               FROM Posts P
                               INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
                               WHERE P.NotFound = 0
-                                AND N.type = 'reply' 
+                                AND N.type = 'reply'
-                                AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')
+                                AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @"
                               GROUP BY P.BlogName, P.PostID
                               ORDER BY LatestTimestamp ASC";
 
@@ -1054,7 +1143,7 @@ namespace URLNotesGrabberCORE
                               INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
                              WHERE N.TimeStamp >= 1535778000
                                AND N.rootBlogName = B.BlogName
-                               AND B.IsActive = 1
+                               AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
                              GROUP BY B.BlogName
                              ORDER BY MIN(N.Timestamp);";
                 }
@@ -1069,7 +1158,7 @@ namespace URLNotesGrabberCORE
                               INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
                              WHERE N.TimeStamp >= 1535778000
                                AND N.rootBlogName = B.BlogName
-                               AND B.IsActive = 1
+                               AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
                                AND (
                                     B.LikesPulled = 0
                                  OR COALESCE(B.LikesLastRefreshed, 0)
@@ -1118,9 +1207,9 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
                 else
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1159,9 +1248,9 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
                 else
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1193,7 +1282,7 @@ namespace URLNotesGrabberCORE
             DBPath ??= GetDefaultDbPath();
             using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             connection.Open();
-            string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
+            string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath);
             using (SQLiteCommand command = new SQLiteCommand(sql, connection))
             {
                 using (SQLiteDataReader reader = command.ExecuteReader())
@@ -1859,6 +1948,8 @@ namespace URLNotesGrabberCORE
             {
                 if (ownsConnection) connection.Open();
 
+                // As in AddPost, IsActive is never written -- neither here nor in the
+                // UPDATE below, which is why an ingest cannot un-remove a post.
                 string insertSql = @"INSERT INTO Posts (
                                 BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
                                 ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
@@ -1989,7 +2080,7 @@ namespace URLNotesGrabberCORE
                                   PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
                                   Question, Answer, Title, PostType,
                                   HasImage, DateCreated, DateModified
-                           FROM Posts WHERE BlogName = @BlogName";
+                           FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath);
             using var cmd = new SQLiteCommand(sql, connection);
             cmd.Parameters.AddWithValue("@BlogName", blogName);
             using var reader = cmd.ExecuteReader();
@@ -2038,7 +2129,7 @@ namespace URLNotesGrabberCORE
                                   PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
                                   Question, Answer, Title, PostType,
                                   HasImage, DateCreated, DateModified
-                           FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID";
+                           FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath);
             using var cmd = new SQLiteCommand(sql, connection);
             cmd.Parameters.AddWithValue("@BlogName", blogName);
             cmd.Parameters.AddWithValue("@PostID", postId);
@@ -2088,7 +2179,7 @@ namespace URLNotesGrabberCORE
                                   PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
                                   Question, Answer, Title, PostType,
                                   HasImage, DateCreated, DateModified
-                           FROM Posts WHERE PostID = @PostID LIMIT 1";
+                           FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1";
             using var cmd = new SQLiteCommand(sql, connection);
             cmd.Parameters.AddWithValue("@PostID", postId);
             using var reader = cmd.ExecuteReader();

URLNotesGrabberCORE/TL.db.md

@@ -262,6 +262,47 @@ under its Posts and Notes pages. Removing a blog hides the blog, not what it col
 
 ---
 
+## `Posts.IsActive` and `Notes.IsActive` — optional, and not in this database yet
+
+The same flag is being extended to the two content tables, with the same meaning: `0` is
+removed, anything else — including `NULL` — is live. **Neither column exists in the live
+`TL.db` as of 2026-07-29**; the DDL quoted above for `Posts` and `Notes` is complete. Like
+`Blogs.IsActive`, they are written from outside this crawler.
+
+The crawler therefore treats both as optional, and as nothing it owns:
+
+- **It never writes them.** No `INSERT` column list names `IsActive`, no `UPDATE` sets it,
+  and `MapPrefixToColumn` — the only place a column name is chosen at runtime — cannot map
+  to it. Re-crawling a removed post or note refreshes its content and leaves the flag at
+  `0`. There is no `INSERT OR REPLACE` on `Posts` or `Notes` for a default to be reset by.
+- **It filters on them only when they exist.** `HasIsActiveColumn` in `DataAccess.cs` asks
+  `PRAGMA table_info` once per table per database path and caches the answer; the filter
+  is `COALESCE(IsActive, 1) = 1`, and it is dropped entirely when the column is absent.
+  Naming a missing column is a hard SQLite error, so this is what lets one build run
+  against databases on both sides of the change. The cache lives for the process — adding
+  the columns to a live database takes effect on the next run.
+
+Every read that selects posts or notes carries the filter: `GetPosts`, `GetReplies`,
+`GetRepliesWithMissingText`, `GetRepliesWithFilledText`, `GetAllPostTextColumns`,
+`GetAllPostsForBlog`, `GetPost`, `GetPostByIdAnyBlog`, and the engagement queries that
+count or join `Notes` (`GetBlogs`, `GetBlogsAll`, `GetBlogsForLikes`). The one deliberate
+omission is the `LEFT JOIN Notes` in `GetPosts`: nothing is selected from it and it can
+neither add nor remove a row, so filtering it would buy nothing.
+
+`LegacyPostsDbImporter` is unfiltered too — it reads a foreign legacy database whose
+`Posts` table is not this schema.
+
+Two consequences worth stating plainly, both inherited from how `Blogs.IsActive` is
+handled:
+
+1. **Removal hides a row; it does not freeze it.** The write paths are keyed on a post the
+   caller already selected, so an ingest or a correction run still overwrites the content
+   of a removed post. Only selection is filtered.
+2. **`NULL` is live.** Write `0` or `1`, not `NULL`, but a `NULL` leaves the row visible
+   rather than stranding it.
+
+---
+
 ## Reproducing the numbers
 
 ```sql

verify-db-schema.sql

@@ -142,6 +142,9 @@ ORDER BY et.tbl;
 -- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
 --     list above. Informational only -- e.g. a NEWER backup, or a column this
 --     script's expected-list hasn't been updated for. Not an error by itself.
+--     Posts.IsActive and Notes.IsActive are listed here and NOT in 1a on
+--     purpose: they are written by other tools, the app only reads them when
+--     present, and it must not be told to add them. See TL.db.md.
 WITH expected(tbl, col) AS (
   VALUES
     ('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
@@ -150,14 +153,14 @@ WITH expected(tbl, col) AS (
     ('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
     ('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
     ('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
-    ('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
+    ('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),('Posts','IsActive'),
     ('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
     ('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
     ('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
     ('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
     ('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
     ('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
-    ('Notes','replyText'),
+    ('Notes','replyText'),('Notes','IsActive'),
     ('DailyAPICount','Date'),('DailyAPICount','APICount'),
     ('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
     ('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')