jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into claude/jovial-mayer-77c2b5

Browse Source
This commit is contained in:
jim
2026-07-29 16:26:50 -05:00
parent 60912c882d 8a4ab2402d
commit 3c85a05afc
4 changed files with 175 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+20
View File
@@ -47,6 +47,26 @@ say nothing about the item being fetched, so they must not be recorded as per-it
- Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or
skipped items), so a caller can distinguish that from a clean run
### `IsActive` Is Not Ours To Write
`Blogs.IsActive`, `Posts.IsActive` and `Notes.IsActive` are removal flags set by other tools
(Rolodex). `0` means removed; anything else, including `NULL`, means live. Full detail in
`URLNotesGrabberCORE/TL.db.md`.
- **Never write any `IsActive` column.** Not in an `INSERT` column list, not in an `UPDATE`,
and never via `INSERT OR REPLACE` on these tables — that resets the column default and
un-removes the row. Re-crawling a removed row must refresh its content and leave the flag
where it was
- **Filter at selection, not at write.** Every query that *selects* posts, notes or blogs
excludes removed rows. Update statements stay keyed on a row the caller already selected;
filtering them would spend API quota and then fail to persist the result
- `Posts.IsActive` and `Notes.IsActive` are **optional** — they are absent from databases
that predate them, and naming a missing column is a hard SQLite error. Compose the filter
with `AndIsActive`/`WhereIsActive` in `DataAccess.cs`, which return
`COALESCE(IsActive, 1) = 1` only when `HasIsActiveColumn` finds the column. `Blogs.IsActive`
is not optional and is filtered directly
- Do not add these columns from this app, and do not add them to the missing-column list in
`verify-db-schema.sql`
### Testing
- No existing test suite; use xUnit if adding tests
- Test critical logic: `ApiKeyPool` init, color parsing, config persistence
URLNotesGrabberCORE/DataAccess.cs
+109 -18
View File
@@ -124,6 +124,91 @@ namespace URLNotesGrabberCORE
return _cachedDbPath;
}
#region IsActive
// Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does:
// 0 = removed elsewhere (Rolodex), anything else (including NULL) = live.
//
// This crawler is a reader of all three. It never writes any IsActive column --
// no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it --
// so a row removed in Rolodex is never resurrected by a re-crawl.
//
// Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added
// from outside this app and are absent from databases that predate them. Naming a
// missing column is a hard SQLite error ("no such column"), so every read asks the
// schema first and simply drops the filter when the column is not there. The answer
// is cached per database path, so adding the columns to a live database takes effect
// on the next run.
private static readonly Dictionary<string, bool> _isActiveColumnCache =
new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
private static readonly object _isActiveColumnLock = new object();
private static bool HasIsActiveColumn(string table, string? DBPath)
{
DBPath ??= GetDefaultDbPath();
string cacheKey = DBPath + "|" + table;
lock (_isActiveColumnLock)
{
if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached))
return cached;
}
bool exists = false;
try
{
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection);
using SQLiteDataReader reader = command.ExecuteReader();
while (reader.Read())
{
if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase))
{
exists = true;
break;
}
}
}
catch (Exception ex)
{
// Breakpoint here
// An unreadable schema is treated as "no column" so the caller's query still runs.
Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}");
}
lock (_isActiveColumnLock)
{
_isActiveColumnCache[cacheKey] = exists;
}
return exists;
}
/// <summary>
/// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it
/// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive.
/// </summary>
private static string AndIsActive(string table, string alias = "", string? DBPath = null)
{
if (!HasIsActiveColumn(table, DBPath)) return string.Empty;
string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + ".";
return $" AND COALESCE({qualifier}IsActive, 1) = 1";
}
/// <summary>
/// Same filter as <see cref="AndIsActive"/>, for a query that has no WHERE clause yet.
/// </summary>
private static string WhereIsActive(string table, string alias = "", string? DBPath = null)
{
string clause = AndIsActive(table, alias, DBPath);
return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length);
}
#endregion IsActive
public static string Q(string input)
{
return "'" + input.Replace("'", "''") + "'";
@@ -464,6 +549,8 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
// IsActive is deliberately absent from this column list: a post removed
// elsewhere must stay removed, so the crawler never writes that flag.
string sql = @"INSERT INTO Posts (
BlogName,
PostID,
@@ -616,6 +703,8 @@ namespace URLNotesGrabberCORE
{
connection2.Open();
// INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note
// that was removed elsewhere leaves the existing row -- and its flag -- alone.
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
{
@@ -713,7 +802,7 @@ namespace URLNotesGrabberCORE
" P.HasNotesGathered," + Environment.NewLine +
" P.NotFound," + Environment.NewLine +
" P.PostDate" + Environment.NewLine +
" FROM Posts P" + Environment.NewLine +
" FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
")," + Environment.NewLine +
"Unioned AS" + Environment.NewLine +
"(" + Environment.NewLine +
@@ -765,8 +854,8 @@ namespace URLNotesGrabberCORE
" LEFT OUTER JOIN " + Environment.NewLine +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine;
" ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
if (beforeDate.HasValue)
{
@@ -875,7 +964,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -918,8 +1007,8 @@ namespace URLNotesGrabberCORE
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
MAX(Notes.timestamp) as LatestTimestamp
FROM Notes
WHERE Notes.type = 'reply'
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
WHERE Notes.type = 'reply'
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @"
GROUP BY Notes.RootBlogName, Notes.PostID
ORDER BY LatestTimestamp ASC
LIMIT @limit";
@@ -966,8 +1055,8 @@ namespace URLNotesGrabberCORE
FROM Posts P
INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
WHERE P.NotFound = 0
AND N.type = 'reply'
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')
AND N.type = 'reply'
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @"
GROUP BY P.BlogName, P.PostID
ORDER BY LatestTimestamp ASC";
@@ -1086,7 +1175,7 @@ namespace URLNotesGrabberCORE
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
AND B.IsActive = 1
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
GROUP BY B.BlogName
ORDER BY MIN(N.Timestamp);";
}
@@ -1101,7 +1190,7 @@ namespace URLNotesGrabberCORE
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
AND B.IsActive = 1
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
AND (
B.LikesPulled = 0
OR COALESCE(B.LikesLastRefreshed, 0)
@@ -1150,9 +1239,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
else
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1191,9 +1280,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
else
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1225,7 +1314,7 @@ namespace URLNotesGrabberCORE
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
@@ -1895,6 +1984,8 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
// As in AddPost, IsActive is never written -- neither here nor in the
// UPDATE below, which is why an ingest cannot un-remove a post.
string insertSql = @"INSERT INTO Posts (
BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
@@ -2025,7 +2116,7 @@ namespace URLNotesGrabberCORE
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE BlogName = @BlogName";
FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath);
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@BlogName", blogName);
using var reader = cmd.ExecuteReader();
@@ -2074,7 +2165,7 @@ namespace URLNotesGrabberCORE
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID";
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath);
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@BlogName", blogName);
cmd.Parameters.AddWithValue("@PostID", postId);
@@ -2124,7 +2215,7 @@ namespace URLNotesGrabberCORE
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE PostID = @PostID LIMIT 1";
FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1";
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@PostID", postId);
using var reader = cmd.ExecuteReader();
URLNotesGrabberCORE/TL.db.md
+41
View File
@@ -262,6 +262,47 @@ under its Posts and Notes pages. Removing a blog hides the blog, not what it col
---
## `Posts.IsActive` and `Notes.IsActive` — optional, and not in this database yet
The same flag is being extended to the two content tables, with the same meaning: `0` is
removed, anything else — including `NULL` — is live. **Neither column exists in the live
`TL.db` as of 2026-07-29**; the DDL quoted above for `Posts` and `Notes` is complete. Like
`Blogs.IsActive`, they are written from outside this crawler.
The crawler therefore treats both as optional, and as nothing it owns:
- **It never writes them.** No `INSERT` column list names `IsActive`, no `UPDATE` sets it,
and `MapPrefixToColumn` — the only place a column name is chosen at runtime — cannot map
to it. Re-crawling a removed post or note refreshes its content and leaves the flag at
`0`. There is no `INSERT OR REPLACE` on `Posts` or `Notes` for a default to be reset by.
- **It filters on them only when they exist.** `HasIsActiveColumn` in `DataAccess.cs` asks
`PRAGMA table_info` once per table per database path and caches the answer; the filter
is `COALESCE(IsActive, 1) = 1`, and it is dropped entirely when the column is absent.
Naming a missing column is a hard SQLite error, so this is what lets one build run
against databases on both sides of the change. The cache lives for the process — adding
the columns to a live database takes effect on the next run.
Every read that selects posts or notes carries the filter: `GetPosts`, `GetReplies`,
`GetRepliesWithMissingText`, `GetRepliesWithFilledText`, `GetAllPostTextColumns`,
`GetAllPostsForBlog`, `GetPost`, `GetPostByIdAnyBlog`, and the engagement queries that
count or join `Notes` (`GetBlogs`, `GetBlogsAll`, `GetBlogsForLikes`). The one deliberate
omission is the `LEFT JOIN Notes` in `GetPosts`: nothing is selected from it and it can
neither add nor remove a row, so filtering it would buy nothing.
`LegacyPostsDbImporter` is unfiltered too — it reads a foreign legacy database whose
`Posts` table is not this schema.
Two consequences worth stating plainly, both inherited from how `Blogs.IsActive` is
handled:
1. **Removal hides a row; it does not freeze it.** The write paths are keyed on a post the
caller already selected, so an ingest or a correction run still overwrites the content
of a removed post. Only selection is filtered.
2. **`NULL` is live.** Write `0` or `1`, not `NULL`, but a `NULL` leaves the row visible
rather than stranding it.
---
## Reproducing the numbers
```sql
verify-db-schema.sql
+5 -2
View File
@@ -142,6 +142,9 @@ ORDER BY et.tbl;
-- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
-- list above. Informational only -- e.g. a NEWER backup, or a column this
-- script's expected-list hasn't been updated for. Not an error by itself.
-- Posts.IsActive and Notes.IsActive are listed here and NOT in 1a on
-- purpose: they are written by other tools, the app only reads them when
-- present, and it must not be told to add them. See TL.db.md.
WITH expected(tbl, col) AS (
VALUES
('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
@@ -150,14 +153,14 @@ WITH expected(tbl, col) AS (
('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),('Posts','IsActive'),
('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
('Notes','replyText'),
('Notes','replyText'),('Notes','IsActive'),
('DailyAPICount','Date'),('DailyAPICount','APICount'),
('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')