Merge branch 'master' into claude/jovial-mayer-77c2b5

The INSERT named a DateCreated column that DailyAPICount (Date, APICount)
never had, so every call threw "no such column: DateCreated" into an
empty catch block. Today's row was never created and the tally sat idle
since 2026-04-13. Drop the column from the INSERT, and report the three
silent failure points (insert error, missing row after insert, update
matching zero rows) instead of swallowing them.

AGENTS.md

@@ -47,6 +47,26 @@ say nothing about the item being fetched, so they must not be recorded as per-it
 - Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or
   skipped items), so a caller can distinguish that from a clean run
 
+### `IsActive` Is Not Ours To Write
+`Blogs.IsActive`, `Posts.IsActive` and `Notes.IsActive` are removal flags set by other tools
+(Rolodex). `0` means removed; anything else, including `NULL`, means live. Full detail in
+`URLNotesGrabberCORE/TL.db.md`.
+
+- **Never write any `IsActive` column.** Not in an `INSERT` column list, not in an `UPDATE`,
+  and never via `INSERT OR REPLACE` on these tables — that resets the column default and
+  un-removes the row. Re-crawling a removed row must refresh its content and leave the flag
+  where it was
+- **Filter at selection, not at write.** Every query that *selects* posts, notes or blogs
+  excludes removed rows. Update statements stay keyed on a row the caller already selected;
+  filtering them would spend API quota and then fail to persist the result
+- `Posts.IsActive` and `Notes.IsActive` are **optional** — they are absent from databases
+  that predate them, and naming a missing column is a hard SQLite error. Compose the filter
+  with `AndIsActive`/`WhereIsActive` in `DataAccess.cs`, which return
+  `COALESCE(IsActive, 1) = 1` only when `HasIsActiveColumn` finds the column. `Blogs.IsActive`
+  is not optional and is filtered directly
+- Do not add these columns from this app, and do not add them to the missing-column list in
+  `verify-db-schema.sql`
+
 ### Testing
 - No existing test suite; use xUnit if adding tests
 - Test critical logic: `ApiKeyPool` init, color parsing, config persistence

URLNotesGrabberCORE/DataAccess.cs

@@ -80,6 +80,10 @@ namespace URLNotesGrabberCORE
         private static SQLiteConnection? _importConnection;
         private static HashSet<string>? _importBlogCache;
         private static readonly object _importSessionLock = new object();
+        // AddAPICount/UpdateAPICount run once per API call. A schema-level failure there repeats
+        // identically every time, so log each distinct message once instead of per call.
+        private static readonly HashSet<string> _apiCountFailuresLogged = new HashSet<string>();
+        private static readonly object _apiCountFailureLock = new object();
 
         static DataAccess()
         {
@@ -120,6 +124,91 @@ namespace URLNotesGrabberCORE
             return _cachedDbPath;
         }
 
+        #region IsActive
+
+        // Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does:
+        // 0 = removed elsewhere (Rolodex), anything else (including NULL) = live.
+        //
+        // This crawler is a reader of all three. It never writes any IsActive column --
+        // no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it --
+        // so a row removed in Rolodex is never resurrected by a re-crawl.
+        //
+        // Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added
+        // from outside this app and are absent from databases that predate them. Naming a
+        // missing column is a hard SQLite error ("no such column"), so every read asks the
+        // schema first and simply drops the filter when the column is not there. The answer
+        // is cached per database path, so adding the columns to a live database takes effect
+        // on the next run.
+        private static readonly Dictionary<string, bool> _isActiveColumnCache =
+            new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
+        private static readonly object _isActiveColumnLock = new object();
+
+        private static bool HasIsActiveColumn(string table, string? DBPath)
+        {
+            DBPath ??= GetDefaultDbPath();
+            string cacheKey = DBPath + "|" + table;
+
+            lock (_isActiveColumnLock)
+            {
+                if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached))
+                    return cached;
+            }
+
+            bool exists = false;
+            try
+            {
+                using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+                connection.Open();
+
+                using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection);
+                using SQLiteDataReader reader = command.ExecuteReader();
+                while (reader.Read())
+                {
+                    if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase))
+                    {
+                        exists = true;
+                        break;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                // Breakpoint here
+                // An unreadable schema is treated as "no column" so the caller's query still runs.
+                Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}");
+            }
+
+            lock (_isActiveColumnLock)
+            {
+                _isActiveColumnCache[cacheKey] = exists;
+            }
+
+            return exists;
+        }
+
+        /// <summary>
+        /// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it
+        /// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive.
+        /// </summary>
+        private static string AndIsActive(string table, string alias = "", string? DBPath = null)
+        {
+            if (!HasIsActiveColumn(table, DBPath)) return string.Empty;
+
+            string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + ".";
+            return $" AND COALESCE({qualifier}IsActive, 1) = 1";
+        }
+
+        /// <summary>
+        /// Same filter as <see cref="AndIsActive"/>, for a query that has no WHERE clause yet.
+        /// </summary>
+        private static string WhereIsActive(string table, string alias = "", string? DBPath = null)
+        {
+            string clause = AndIsActive(table, alias, DBPath);
+            return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length);
+        }
+
+        #endregion IsActive
+
         public static string Q(string input)
         {
             return "'" + input.Replace("'", "''") + "'";
@@ -460,6 +549,8 @@ namespace URLNotesGrabberCORE
             {
                 if (ownsConnection) connection.Open();
 
+                // IsActive is deliberately absent from this column list: a post removed
+                // elsewhere must stay removed, so the crawler never writes that flag.
                 string sql = @"INSERT INTO Posts (
                                 BlogName,
                                 PostID,
@@ -566,21 +657,40 @@ namespace URLNotesGrabberCORE
 
                 // Use INSERT OR IGNORE to avoid UNIQUE constraint errors when the date row already exists.
                 // Also explicitly initialize APICount to 0 in case the table has no default.
-                string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount, DateCreated) values(@date, 0, @DateCreated)";
+                //
+                // DailyAPICount is (Date TEXT PK, APICount INTEGER) — the crawler never creates or
+                // migrates this table, and no code reads a creation timestamp off it, so the insert
+                // names only those two columns. Naming a DateCreated column here used to throw
+                // "no such column: DateCreated" into a silent catch, which meant the day's row was
+                // never created and the tally sat at 0 for months.
+                string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount) values(@date, 0)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
-                    command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                     command.ExecuteNonQuery();
                 }
             }
             catch (Exception ex)
             {
                 // Breakpoint here
-                //Console.WriteLine(ex.Message);
+                ReportAPICountFailure($"Error creating the row for {DateTime.Today.ToShortDateString()}: {ex.Message}");
             }
         }
 
+        // Bookkeeping writes that fail identically on every API call would flood the console, but
+        // swallowing them entirely is what hid the DateCreated bug. Log each distinct message once.
+        // Messages embed today's date, so a date rollover reports afresh.
+        private static void ReportAPICountFailure(string message)
+        {
+            lock (_apiCountFailureLock)
+            {
+                if (!_apiCountFailuresLogged.Add(message))
+                    return;
+            }
+
+            Console.WriteLine($"[DailyAPICount] {message}");
+        }
+
         public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
@@ -593,6 +703,8 @@ namespace URLNotesGrabberCORE
             {
                 connection2.Open();
 
+                // INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note
+                // that was removed elsewhere leaves the existing row -- and its flag -- alone.
                 string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
                 {
@@ -690,7 +802,7 @@ namespace URLNotesGrabberCORE
                           "        P.HasNotesGathered," + Environment.NewLine +
                           "        P.NotFound," + Environment.NewLine +
                           "        P.PostDate" + Environment.NewLine +
-                          "    FROM Posts P" + Environment.NewLine +
+                          "    FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
                           ")," + Environment.NewLine +
                           "Unioned AS" + Environment.NewLine +
                           "(" + Environment.NewLine +
@@ -742,8 +854,8 @@ namespace URLNotesGrabberCORE
                                  "      LEFT OUTER JOIN " + Environment.NewLine +
                                  "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
                                  "      LEFT OUTER JOIN " + Environment.NewLine +
-                                 "   (      select BlogName, count(PostID) as CNT from Posts group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
+                                 "   (      select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
-                                 "WHERE NotFound = 0 " + Environment.NewLine;
+                                 "WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
 
                     if (beforeDate.HasValue)
                     {
@@ -852,7 +964,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "SELECT	distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
+                string sql = "SELECT	distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -895,8 +1007,8 @@ namespace URLNotesGrabberCORE
                 string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
                              MAX(Notes.timestamp) as LatestTimestamp
                              FROM Notes 
-                             WHERE Notes.type = 'reply' 
+                             WHERE Notes.type = 'reply'
-                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @"
                              GROUP BY Notes.RootBlogName, Notes.PostID
                              ORDER BY LatestTimestamp ASC
                              LIMIT @limit";
@@ -943,8 +1055,8 @@ namespace URLNotesGrabberCORE
                               FROM Posts P
                               INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
                               WHERE P.NotFound = 0
-                                AND N.type = 'reply' 
+                                AND N.type = 'reply'
-                                AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')
+                                AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @"
                               GROUP BY P.BlogName, P.PostID
                               ORDER BY LatestTimestamp ASC";
 
@@ -993,7 +1105,9 @@ namespace URLNotesGrabberCORE
             using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             int count = 0;
 
-            try { AddAPICount(); } catch { }
+            // AddAPICount reports its own failures; this guard only stops a connection-level
+            // problem from taking down the read below.
+            try { AddAPICount(); } catch (Exception ex) { ReportAPICountFailure($"AddAPICount failed: {ex.Message}"); }
 
             try
             {
@@ -1001,6 +1115,7 @@ namespace URLNotesGrabberCORE
 
                 string sql = "SELECT APICount FROM DailyAPICount WHERE [Date] = @date";
 
+                bool rowFound = false;
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
@@ -1008,10 +1123,16 @@ namespace URLNotesGrabberCORE
                     {
                         while (reader.Read())
                         {
+                            rowFound = true;
                             count = reader.GetInt32(0); // Assuming Id is the first column
                         }
                     }
                 }
+
+                // A missing row means AddAPICount did not take. Returning a silent 0 here is what
+                // made the tally look merely idle rather than broken.
+                if (!rowFound)
+                    ReportAPICountFailure($"No row for {DateTime.Today.ToShortDateString()} after AddAPICount - reported count of 0 is not a real tally.");
             }
             catch (Exception ex)
             {
@@ -1054,7 +1175,7 @@ namespace URLNotesGrabberCORE
                               INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
                              WHERE N.TimeStamp >= 1535778000
                                AND N.rootBlogName = B.BlogName
-                               AND B.IsActive = 1
+                               AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
                              GROUP BY B.BlogName
                              ORDER BY MIN(N.Timestamp);";
                 }
@@ -1069,7 +1190,7 @@ namespace URLNotesGrabberCORE
                               INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
                              WHERE N.TimeStamp >= 1535778000
                                AND N.rootBlogName = B.BlogName
-                               AND B.IsActive = 1
+                               AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
                                AND (
                                     B.LikesPulled = 0
                                  OR COALESCE(B.LikesLastRefreshed, 0)
@@ -1118,9 +1239,9 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
                 else
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1159,9 +1280,9 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
                 else
-                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
+                    sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1193,7 +1314,7 @@ namespace URLNotesGrabberCORE
             DBPath ??= GetDefaultDbPath();
             using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             connection.Open();
-            string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
+            string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath);
             using (SQLiteCommand command = new SQLiteCommand(sql, connection))
             {
                 using (SQLiteDataReader reader = command.ExecuteReader())
@@ -1654,7 +1775,11 @@ namespace URLNotesGrabberCORE
                 {
                     command.Parameters.AddWithValue("@APICount", APICount);
                     command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
-                    command.ExecuteNonQuery();
+
+                    // No row for today means this UPDATE matched nothing and the increment was
+                    // thrown away, while the value returned below still looks like a real count.
+                    if (command.ExecuteNonQuery() == 0)
+                        ReportAPICountFailure($"UPDATE matched no row for {DateTime.Today.ToShortDateString()} - the count of {APICount} was not persisted.");
                 }
             }
             catch (Exception ex)
@@ -1859,6 +1984,8 @@ namespace URLNotesGrabberCORE
             {
                 if (ownsConnection) connection.Open();
 
+                // As in AddPost, IsActive is never written -- neither here nor in the
+                // UPDATE below, which is why an ingest cannot un-remove a post.
                 string insertSql = @"INSERT INTO Posts (
                                 BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
                                 ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
@@ -1989,7 +2116,7 @@ namespace URLNotesGrabberCORE
                                   PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
                                   Question, Answer, Title, PostType,
                                   HasImage, DateCreated, DateModified
-                           FROM Posts WHERE BlogName = @BlogName";
+                           FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath);
             using var cmd = new SQLiteCommand(sql, connection);
             cmd.Parameters.AddWithValue("@BlogName", blogName);
             using var reader = cmd.ExecuteReader();
@@ -2038,7 +2165,7 @@ namespace URLNotesGrabberCORE
                                   PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
                                   Question, Answer, Title, PostType,
                                   HasImage, DateCreated, DateModified
-                           FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID";
+                           FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath);
             using var cmd = new SQLiteCommand(sql, connection);
             cmd.Parameters.AddWithValue("@BlogName", blogName);
             cmd.Parameters.AddWithValue("@PostID", postId);
@@ -2088,7 +2215,7 @@ namespace URLNotesGrabberCORE
                                   PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
                                   Question, Answer, Title, PostType,
                                   HasImage, DateCreated, DateModified
-                           FROM Posts WHERE PostID = @PostID LIMIT 1";
+                           FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1";
             using var cmd = new SQLiteCommand(sql, connection);
             cmd.Parameters.AddWithValue("@PostID", postId);
             using var reader = cmd.ExecuteReader();

URLNotesGrabberCORE/TL.db.md

@@ -0,0 +1,333 @@
+# `TL.db` — schema notes
+
+The SQLite database behind **URLNotesGrabberCORE** and its sibling crawlers, and the one
+[Rolodex](https://git.basso.land/jim/Rolodex) reads.
+
+Everything below was read out of the live file, not inferred from code. Counts are as of
+**2026-07-29**; re-run the queries at the bottom to refresh them.
+
+- Journal mode: **WAL** — `TL.db-wal` and `TL.db-shm` live beside the file and are part of
+  the database. Copying `TL.db` alone gives you whatever was last checkpointed, not the
+  current state.
+- Page size: 4096.
+
+---
+
+## The three content tables
+
+| Table | Rows | What it is |
+|---|--:|---|
+| `Blogs` | 144,367 | The crawl registry — one row per known blog, plus crawl-state flags |
+| `Posts` | 14,589 | Stored post content. Only 3,602 blogs actually have any |
+| `Notes` | 1,189,604 | The engagement graph: `NoteBlogName` acted on `(RootBlogName, PostID)` |
+
+The engagement graph is the interesting part. 31,888 distinct blogs appear as engagers —
+far more than the 3,602 that have stored posts — which is what makes this a social graph
+rather than a post archive.
+
+### `Blogs`
+
+```sql
+CREATE TABLE "Blogs" (
+    "BlogName"              TEXT,
+    "HasBeenOutput"         INTEGER DEFAULT 0,
+    "IsActive"              INTEGER DEFAULT 1,
+    "DateAdded"             TEXT NOT NULL DEFAULT '12/24/25',
+    "ByLikes"               INTEGER NOT NULL DEFAULT 0,
+    "LikesPulled"           INTEGER NOT NULL DEFAULT 0,
+    "LikesCursor"           INTEGER DEFAULT 0,
+    "DateModified"          TEXT,
+    "DateCreated"           TEXT,
+    LikesNewestTimestamp    INTEGER DEFAULT 0,
+    LikesLastRefreshed      INTEGER DEFAULT 0,
+    LikesLastNewCount       INTEGER DEFAULT 0,
+    TTFolderPath            TEXT,
+    PRIMARY KEY("BlogName")
+);
+```
+
+`BlogName` is the primary key, so it is the only indexed way in. There is no index on any
+flag or date — filtering or sorting on those scans all 144k rows, which is affordable
+here and is not on `Notes`.
+
+Flag distribution: `IsActive = 1` on 144,366 of 144,367 rows, `HasBeenOutput = 1` on
+5,369, `ByLikes = 1` on 2. `IsActive` carries a second meaning as of Rolodex — see
+[`Blogs.IsActive`](#blogsisactive--now-written-by-two-applications) below.
+
+The columns after `DateCreated` were added later by `ALTER TABLE`, which is why they carry
+no quoting in the stored DDL. That is the normal way this schema grows.
+
+**`DateAdded` is not written consistently.** 126,423 rows hold ISO `yyyy-MM-dd HH:mm:ss`;
+17,944 hold US-format `M/d/yy` from a bulk import. As text those two sort into different
+parts of the table, so anything ordering or range-filtering on this column has to
+normalise first — see `DateSql` in Rolodex.
+
+### `Posts`
+
+```sql
+CREATE TABLE "Posts" (
+    "BlogName"              TEXT,
+    "PostID"                INTEGER,
+    "HasNotesGathered"      INTEGER DEFAULT 0,
+    "reblogURL"             TEXT,
+    "NotFound"              INTEGER DEFAULT 0,
+    "PostDate"              TEXT,
+    "NotesGatheredDateTime" INTEGER NOT NULL DEFAULT 1729746000,
+    "HasImage"              INTEGER NOT NULL DEFAULT 0,
+    "PostURL"               TEXT,
+    "Slug"                  TEXT,
+    "ReblogKey"             TEXT,
+    "ReblogName"            TEXT,
+    "Summary"               TEXT,
+    "Quote"                 TEXT,
+    "Body"                  TEXT,
+    "Tags"                  TEXT,
+    "Link"                  TEXT,
+    "PhotoURL"              TEXT,
+    "PhotoCaption"          TEXT,
+    "DownloadedFiles"       TEXT,
+    "AudioCaption"          TEXT,
+    "Question"              TEXT,
+    "Answer"                TEXT,
+    "Title"                 TEXT,
+    "ByLikes"               INTEGER NOT NULL DEFAULT 0,
+    "RootBlogName"          TEXT,
+    "RootURL"               TEXT,
+    "DateModified"          TEXT,
+    "DateCreated"           TEXT,
+    PostType                TEXT,
+    PRIMARY KEY("BlogName","PostID")
+);
+```
+
+**The key is `(BlogName, PostID)`, not `PostID`.** This matters more than it looks: 325
+post IDs exist under more than one blog, so an ID on its own is both ambiguous *and*
+unindexed. Any lookup should carry the blog name, and a batch lookup should group by blog
+so it stays on the leading column of the key.
+
+Notable:
+
+- **`PostType` is `NULL` on all 14,589 rows.** The column exists but nothing has ever
+  populated it. Treat it as unpopulated rather than as a type discriminator.
+- `HasImage = 1` on 14,268 rows — nearly all of them. It records that the post *had* a
+  picture, not that a usable URL was kept, so it is not a reliable predictor that anything
+  will render.
+- `PhotoURL` is largely unused; in practice the image markup lives inside `Body`.
+- `NotFound = 1` on 4,663 rows — posts that have since been deleted upstream.
+- The content columns (`Body`, `Quote`, `Question`, `Answer`, …) are the heavy ones. List
+  views should not select them.
+
+### `Notes`
+
+```sql
+CREATE TABLE "Notes" (
+    "RootBlogName"    TEXT,
+    "PostID"          INTEGER,
+    "NoteBlogName"    TEXT,
+    "TimeStamp"       INTEGER,
+    "Type"            TEXT,
+    "replyText"       TEXT DEFAULT '.',
+    "DatetimeCrawled" TEXT DEFAULT '2/12/26 12am',
+    "DateModified"    TEXT,
+    "DateCreated"     TEXT,
+    PRIMARY KEY("RootBlogName","PostID","TimeStamp","Type","NoteBlogName")
+);
+
+CREATE INDEX "Notes_idx_06e01ae3" ON "Notes" ("TimeStamp" DESC);
+CREATE INDEX "ix_NoteBlogName01"  ON "Notes" ("NoteBlogName");
+```
+
+One row per engagement event. `TimeStamp` is **unix seconds** — unlike every date column
+elsewhere in the schema, which are text.
+
+| `Type` | Rows | Share |
+|---|--:|--:|
+| `like` | 947,955 | 79.7% |
+| `reblog` | 224,323 | 18.9% |
+| `reply` | 15,201 | 1.3% |
+| `posted` | 2,106 | 0.2% |
+| `post_attribution` | 19 | — |
+
+At 1.19M rows this is the table that dictates how the whole database has to be queried:
+
+- **Nothing should run an unbounded `SELECT` or a bare `COUNT(*)` here.** A count scans
+  the lot on every call.
+- The only fast access paths are the primary key's leading columns (`RootBlogName`, then
+  `PostID`) and `ix_NoteBlogName01` on `NoteBlogName`. "Notes received by a blog" and
+  "notes given by a blog" are both cheap; almost nothing else is.
+- Ordering by anything but `TimeStamp` is a full sort of whatever the filters leave.
+- `replyText` is `'.'` on 1,174,706 rows — only `reply` notes carry real text.
+
+### Referential integrity
+
+There are no foreign keys, and the tables do not perfectly agree:
+
+- 4 `Posts` rows name a blog with no `Blogs` row.
+- 15 of the 31,888 distinct engagers have no `Blogs` row.
+
+So a name appearing in `Notes` or `Posts` is not a guarantee that the registry knows about
+it. Joins from those tables back to `Blogs` should tolerate a miss.
+
+---
+
+## The `'.'` placeholder convention
+
+**The crawler writes a single dot into text columns it has no value for, rather than
+`NULL`.** This is the single most surprising thing about the schema and it affects every
+consumer.
+
+| Column | `'.'` rows |
+|---|--:|
+| `Notes.replyText` | 1,174,706 |
+| `Posts.Title` | 13,144 |
+| `Posts.Body` | 172 |
+
+Any query whose output reaches a human should collapse it:
+
+```sql
+NULLIF(NULLIF(SomeColumn, '.'), '') AS SomeColumn
+```
+
+Empty string turns up too, hence the double `NULLIF`. Not every column is affected —
+`Blogs.TTFolderPath` and `Blogs.DateModified` currently have zero dot rows — but new
+columns tend to acquire them, so treat cleaning as the default for any text column
+rendered to a user.
+
+---
+
+## Supporting tables
+
+Crawler bookkeeping. Rolodex ignores all of these.
+
+| Table | Rows | What it is |
+|---|--:|---|
+| `DailyAPICount` | 133 | `(Date TEXT PK, APICount INTEGER)` — per-day API call tally against the rate limit |
+| `ApiKeyPoolState` | 2 | `(KeyName TEXT PK, RetryUntil INTEGER)` — per-key backoff; `RetryUntil` is unix seconds |
+| `ApiKeyPoolMeta` | 1 | `(Id PK CHECK (Id = 1), LastIndex)` — round-robin cursor. Singleton by check constraint |
+| `CollectRunState` | 1 | `(Id PK CHECK (Id = 1), RunCutoff, RunComplete, RunStarted, RunCompletedAt)` — resume state for an interrupted collection run. Also a singleton |
+
+---
+
+## `Blogs.IsActive` — now written by two applications
+
+`IsActive` has always been the crawler's work-selection flag. `GetBlogs` in
+`DataAccess.cs` joins on it to decide what to collect:
+
+```sql
+SELECT NoteBlogName, count(*) FROM notes
+INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName
+WHERE blogs.IsActive = @isActive AND ...
+```
+
+Nothing inside the crawler *writes* it — it is an input, set from outside.
+
+**Rolodex is now one of the things that sets it.** Removing a blog through the Rolodex UI
+runs exactly this:
+
+```sql
+UPDATE Blogs SET IsActive = 0 WHERE BlogName = ?;
+```
+
+Rolodex adds no column and changes no schema. It reuses this flag because the two meanings
+were judged to be one decision: a blog you do not want in the browsing UI is a blog you do
+not want to keep crawling. Removal therefore stops collection, and the Rolodex
+confirmation screen says so before anyone commits.
+
+- `1` (or absent/NULL) — live. Crawled, and visible in Rolodex.
+- `0` — removed. Not crawled, hidden from the Rolodex registry, dashboard counts and
+  engagement rollups.
+
+Restoring is the same `UPDATE` with a `1`. Nothing is destroyed either way: the blog's
+`Posts` and `Notes` rows are never touched, and Rolodex deliberately keeps showing them
+under its Posts and Notes pages. Removing a blog hides the blog, not what it collected.
+
+### What other tools need to know
+
+1. **Setting `IsActive = 0` now also hides the blog from Rolodex**, and setting it back to
+   `1` makes it reappear. If another tool deactivates blogs in bulk, it is also removing
+   them from the browsing UI — which may be exactly right, but it is no longer a
+   crawler-only decision.
+2. **Re-crawling a removed blog will not bring it back**, since nothing in the crawler
+   writes the flag. An `INSERT OR REPLACE` on the `Blogs` row *would*, by resetting it to
+   the column default of `1`. Prefer an `UPDATE` of the specific columns, or
+   `INSERT … ON CONFLICT DO UPDATE SET` naming only the columns being refreshed.
+3. **NULL is treated as live.** The column is `INTEGER DEFAULT 1` with no `NOT NULL`, so
+   Rolodex reads it through `COALESCE(IsActive, 1)`. A NULL therefore leaves the blog
+   visible rather than stranding it outside both the registry and the removed list, where
+   no screen could reach it. Write `0` or `1`, not NULL.
+4. **Backing the feature out is a configuration change, not a migration.** Because there is
+   no Rolodex-owned column, setting `Rolodex__EnableBlogDeletion=false` is the whole of it;
+   there is nothing to drop. Any blogs already at `IsActive = 0` simply go back to being
+   ordinary inactive blogs.
+
+---
+
+## `Posts.IsActive` and `Notes.IsActive` — optional, and not in this database yet
+
+The same flag is being extended to the two content tables, with the same meaning: `0` is
+removed, anything else — including `NULL` — is live. **Neither column exists in the live
+`TL.db` as of 2026-07-29**; the DDL quoted above for `Posts` and `Notes` is complete. Like
+`Blogs.IsActive`, they are written from outside this crawler.
+
+The crawler therefore treats both as optional, and as nothing it owns:
+
+- **It never writes them.** No `INSERT` column list names `IsActive`, no `UPDATE` sets it,
+  and `MapPrefixToColumn` — the only place a column name is chosen at runtime — cannot map
+  to it. Re-crawling a removed post or note refreshes its content and leaves the flag at
+  `0`. There is no `INSERT OR REPLACE` on `Posts` or `Notes` for a default to be reset by.
+- **It filters on them only when they exist.** `HasIsActiveColumn` in `DataAccess.cs` asks
+  `PRAGMA table_info` once per table per database path and caches the answer; the filter
+  is `COALESCE(IsActive, 1) = 1`, and it is dropped entirely when the column is absent.
+  Naming a missing column is a hard SQLite error, so this is what lets one build run
+  against databases on both sides of the change. The cache lives for the process — adding
+  the columns to a live database takes effect on the next run.
+
+Every read that selects posts or notes carries the filter: `GetPosts`, `GetReplies`,
+`GetRepliesWithMissingText`, `GetRepliesWithFilledText`, `GetAllPostTextColumns`,
+`GetAllPostsForBlog`, `GetPost`, `GetPostByIdAnyBlog`, and the engagement queries that
+count or join `Notes` (`GetBlogs`, `GetBlogsAll`, `GetBlogsForLikes`). The one deliberate
+omission is the `LEFT JOIN Notes` in `GetPosts`: nothing is selected from it and it can
+neither add nor remove a row, so filtering it would buy nothing.
+
+`LegacyPostsDbImporter` is unfiltered too — it reads a foreign legacy database whose
+`Posts` table is not this schema.
+
+Two consequences worth stating plainly, both inherited from how `Blogs.IsActive` is
+handled:
+
+1. **Removal hides a row; it does not freeze it.** The write paths are keyed on a post the
+   caller already selected, so an ingest or a correction run still overwrites the content
+   of a removed post. Only selection is filtered.
+2. **`NULL` is live.** Write `0` or `1`, not `NULL`, but a `NULL` leaves the row visible
+   rather than stranding it.
+
+---
+
+## Reproducing the numbers
+
+```sql
+SELECT 'Blogs', COUNT(*) FROM Blogs
+UNION ALL SELECT 'Posts', COUNT(*) FROM Posts
+UNION ALL SELECT 'Notes', COUNT(*) FROM Notes;
+
+-- note type mix
+SELECT Type, COUNT(*) FROM Notes GROUP BY Type ORDER BY 2 DESC;
+
+-- the two date shapes in Blogs.DateAdded
+SELECT CASE WHEN DateAdded LIKE '____-__-__%' THEN 'ISO' ELSE 'US' END, COUNT(*)
+FROM Blogs GROUP BY 1;
+
+-- post IDs that are ambiguous without a blog name
+SELECT COUNT(*) FROM (
+    SELECT PostID FROM Posts GROUP BY PostID HAVING COUNT(DISTINCT BlogName) > 1);
+
+-- rows that reference a blog the registry does not have
+SELECT COUNT(*) FROM Posts p
+WHERE NOT EXISTS (SELECT 1 FROM Blogs b WHERE b.BlogName = p.BlogName);
+```
+
+Open the file read-only so an inspection can never disturb a running crawl:
+
+```bash
+sqlite3 "file:TL.db?mode=ro" ".schema"
+```

verify-db-schema.sql

@@ -142,6 +142,9 @@ ORDER BY et.tbl;
 -- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
 --     list above. Informational only -- e.g. a NEWER backup, or a column this
 --     script's expected-list hasn't been updated for. Not an error by itself.
+--     Posts.IsActive and Notes.IsActive are listed here and NOT in 1a on
+--     purpose: they are written by other tools, the app only reads them when
+--     present, and it must not be told to add them. See TL.db.md.
 WITH expected(tbl, col) AS (
   VALUES
     ('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
@@ -150,14 +153,14 @@ WITH expected(tbl, col) AS (
     ('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
     ('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
     ('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
-    ('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
+    ('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),('Posts','IsActive'),
     ('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
     ('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
     ('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
     ('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
     ('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
     ('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
-    ('Notes','replyText'),
+    ('Notes','replyText'),('Notes','IsActive'),
     ('DailyAPICount','Date'),('DailyAPICount','APICount'),
     ('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
     ('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')