Compare commits
2
Commits
eded5271ea
...
8a4ab2402d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8a4ab2402d | ||
|
|
05ec465f74 |
@@ -47,6 +47,26 @@ say nothing about the item being fetched, so they must not be recorded as per-it
|
|||||||
- Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or
|
- Long-running commands return exit 3 when a pass ends incomplete (rate-limit pause, breaker trip, or
|
||||||
skipped items), so a caller can distinguish that from a clean run
|
skipped items), so a caller can distinguish that from a clean run
|
||||||
|
|
||||||
|
### `IsActive` Is Not Ours To Write
|
||||||
|
`Blogs.IsActive`, `Posts.IsActive` and `Notes.IsActive` are removal flags set by other tools
|
||||||
|
(Rolodex). `0` means removed; anything else, including `NULL`, means live. Full detail in
|
||||||
|
`URLNotesGrabberCORE/TL.db.md`.
|
||||||
|
|
||||||
|
- **Never write any `IsActive` column.** Not in an `INSERT` column list, not in an `UPDATE`,
|
||||||
|
and never via `INSERT OR REPLACE` on these tables — that resets the column default and
|
||||||
|
un-removes the row. Re-crawling a removed row must refresh its content and leave the flag
|
||||||
|
where it was
|
||||||
|
- **Filter at selection, not at write.** Every query that *selects* posts, notes or blogs
|
||||||
|
excludes removed rows. Update statements stay keyed on a row the caller already selected;
|
||||||
|
filtering them would spend API quota and then fail to persist the result
|
||||||
|
- `Posts.IsActive` and `Notes.IsActive` are **optional** — they are absent from databases
|
||||||
|
that predate them, and naming a missing column is a hard SQLite error. Compose the filter
|
||||||
|
with `AndIsActive`/`WhereIsActive` in `DataAccess.cs`, which return
|
||||||
|
`COALESCE(IsActive, 1) = 1` only when `HasIsActiveColumn` finds the column. `Blogs.IsActive`
|
||||||
|
is not optional and is filtered directly
|
||||||
|
- Do not add these columns from this app, and do not add them to the missing-column list in
|
||||||
|
`verify-db-schema.sql`
|
||||||
|
|
||||||
### Testing
|
### Testing
|
||||||
- No existing test suite; use xUnit if adding tests
|
- No existing test suite; use xUnit if adding tests
|
||||||
- Test critical logic: `ApiKeyPool` init, color parsing, config persistence
|
- Test critical logic: `ApiKeyPool` init, color parsing, config persistence
|
||||||
|
|||||||
@@ -120,6 +120,91 @@ namespace URLNotesGrabberCORE
|
|||||||
return _cachedDbPath;
|
return _cachedDbPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#region IsActive
|
||||||
|
|
||||||
|
// Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does:
|
||||||
|
// 0 = removed elsewhere (Rolodex), anything else (including NULL) = live.
|
||||||
|
//
|
||||||
|
// This crawler is a reader of all three. It never writes any IsActive column --
|
||||||
|
// no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it --
|
||||||
|
// so a row removed in Rolodex is never resurrected by a re-crawl.
|
||||||
|
//
|
||||||
|
// Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added
|
||||||
|
// from outside this app and are absent from databases that predate them. Naming a
|
||||||
|
// missing column is a hard SQLite error ("no such column"), so every read asks the
|
||||||
|
// schema first and simply drops the filter when the column is not there. The answer
|
||||||
|
// is cached per database path, so adding the columns to a live database takes effect
|
||||||
|
// on the next run.
|
||||||
|
private static readonly Dictionary<string, bool> _isActiveColumnCache =
|
||||||
|
new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
private static readonly object _isActiveColumnLock = new object();
|
||||||
|
|
||||||
|
private static bool HasIsActiveColumn(string table, string? DBPath)
|
||||||
|
{
|
||||||
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
string cacheKey = DBPath + "|" + table;
|
||||||
|
|
||||||
|
lock (_isActiveColumnLock)
|
||||||
|
{
|
||||||
|
if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached))
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool exists = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
|
connection.Open();
|
||||||
|
|
||||||
|
using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection);
|
||||||
|
using SQLiteDataReader reader = command.ExecuteReader();
|
||||||
|
while (reader.Read())
|
||||||
|
{
|
||||||
|
if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
exists = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// Breakpoint here
|
||||||
|
// An unreadable schema is treated as "no column" so the caller's query still runs.
|
||||||
|
Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
lock (_isActiveColumnLock)
|
||||||
|
{
|
||||||
|
_isActiveColumnCache[cacheKey] = exists;
|
||||||
|
}
|
||||||
|
|
||||||
|
return exists;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it
|
||||||
|
/// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive.
|
||||||
|
/// </summary>
|
||||||
|
private static string AndIsActive(string table, string alias = "", string? DBPath = null)
|
||||||
|
{
|
||||||
|
if (!HasIsActiveColumn(table, DBPath)) return string.Empty;
|
||||||
|
|
||||||
|
string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + ".";
|
||||||
|
return $" AND COALESCE({qualifier}IsActive, 1) = 1";
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Same filter as <see cref="AndIsActive"/>, for a query that has no WHERE clause yet.
|
||||||
|
/// </summary>
|
||||||
|
private static string WhereIsActive(string table, string alias = "", string? DBPath = null)
|
||||||
|
{
|
||||||
|
string clause = AndIsActive(table, alias, DBPath);
|
||||||
|
return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion IsActive
|
||||||
|
|
||||||
public static string Q(string input)
|
public static string Q(string input)
|
||||||
{
|
{
|
||||||
return "'" + input.Replace("'", "''") + "'";
|
return "'" + input.Replace("'", "''") + "'";
|
||||||
@@ -460,6 +545,8 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
if (ownsConnection) connection.Open();
|
if (ownsConnection) connection.Open();
|
||||||
|
|
||||||
|
// IsActive is deliberately absent from this column list: a post removed
|
||||||
|
// elsewhere must stay removed, so the crawler never writes that flag.
|
||||||
string sql = @"INSERT INTO Posts (
|
string sql = @"INSERT INTO Posts (
|
||||||
BlogName,
|
BlogName,
|
||||||
PostID,
|
PostID,
|
||||||
@@ -593,6 +680,8 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
connection2.Open();
|
connection2.Open();
|
||||||
|
|
||||||
|
// INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note
|
||||||
|
// that was removed elsewhere leaves the existing row -- and its flag -- alone.
|
||||||
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
|
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
|
||||||
{
|
{
|
||||||
@@ -690,7 +779,7 @@ namespace URLNotesGrabberCORE
|
|||||||
" P.HasNotesGathered," + Environment.NewLine +
|
" P.HasNotesGathered," + Environment.NewLine +
|
||||||
" P.NotFound," + Environment.NewLine +
|
" P.NotFound," + Environment.NewLine +
|
||||||
" P.PostDate" + Environment.NewLine +
|
" P.PostDate" + Environment.NewLine +
|
||||||
" FROM Posts P" + Environment.NewLine +
|
" FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
|
||||||
")," + Environment.NewLine +
|
")," + Environment.NewLine +
|
||||||
"Unioned AS" + Environment.NewLine +
|
"Unioned AS" + Environment.NewLine +
|
||||||
"(" + Environment.NewLine +
|
"(" + Environment.NewLine +
|
||||||
@@ -742,8 +831,8 @@ namespace URLNotesGrabberCORE
|
|||||||
" LEFT OUTER JOIN " + Environment.NewLine +
|
" LEFT OUTER JOIN " + Environment.NewLine +
|
||||||
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
|
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
|
||||||
" LEFT OUTER JOIN " + Environment.NewLine +
|
" LEFT OUTER JOIN " + Environment.NewLine +
|
||||||
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
|
" ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
|
||||||
"WHERE NotFound = 0 " + Environment.NewLine;
|
"WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
|
||||||
|
|
||||||
if (beforeDate.HasValue)
|
if (beforeDate.HasValue)
|
||||||
{
|
{
|
||||||
@@ -852,7 +941,7 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
connection.Open();
|
connection.Open();
|
||||||
|
|
||||||
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
|
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID";
|
||||||
|
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
@@ -895,8 +984,8 @@ namespace URLNotesGrabberCORE
|
|||||||
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
|
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
|
||||||
MAX(Notes.timestamp) as LatestTimestamp
|
MAX(Notes.timestamp) as LatestTimestamp
|
||||||
FROM Notes
|
FROM Notes
|
||||||
WHERE Notes.type = 'reply'
|
WHERE Notes.type = 'reply'
|
||||||
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
|
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @"
|
||||||
GROUP BY Notes.RootBlogName, Notes.PostID
|
GROUP BY Notes.RootBlogName, Notes.PostID
|
||||||
ORDER BY LatestTimestamp ASC
|
ORDER BY LatestTimestamp ASC
|
||||||
LIMIT @limit";
|
LIMIT @limit";
|
||||||
@@ -943,8 +1032,8 @@ namespace URLNotesGrabberCORE
|
|||||||
FROM Posts P
|
FROM Posts P
|
||||||
INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
|
INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
|
||||||
WHERE P.NotFound = 0
|
WHERE P.NotFound = 0
|
||||||
AND N.type = 'reply'
|
AND N.type = 'reply'
|
||||||
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')
|
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @"
|
||||||
GROUP BY P.BlogName, P.PostID
|
GROUP BY P.BlogName, P.PostID
|
||||||
ORDER BY LatestTimestamp ASC";
|
ORDER BY LatestTimestamp ASC";
|
||||||
|
|
||||||
@@ -1054,7 +1143,7 @@ namespace URLNotesGrabberCORE
|
|||||||
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
|
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
|
||||||
WHERE N.TimeStamp >= 1535778000
|
WHERE N.TimeStamp >= 1535778000
|
||||||
AND N.rootBlogName = B.BlogName
|
AND N.rootBlogName = B.BlogName
|
||||||
AND B.IsActive = 1
|
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
|
||||||
GROUP BY B.BlogName
|
GROUP BY B.BlogName
|
||||||
ORDER BY MIN(N.Timestamp);";
|
ORDER BY MIN(N.Timestamp);";
|
||||||
}
|
}
|
||||||
@@ -1069,7 +1158,7 @@ namespace URLNotesGrabberCORE
|
|||||||
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
|
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
|
||||||
WHERE N.TimeStamp >= 1535778000
|
WHERE N.TimeStamp >= 1535778000
|
||||||
AND N.rootBlogName = B.BlogName
|
AND N.rootBlogName = B.BlogName
|
||||||
AND B.IsActive = 1
|
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
|
||||||
AND (
|
AND (
|
||||||
B.LikesPulled = 0
|
B.LikesPulled = 0
|
||||||
OR COALESCE(B.LikesLastRefreshed, 0)
|
OR COALESCE(B.LikesLastRefreshed, 0)
|
||||||
@@ -1118,9 +1207,9 @@ namespace URLNotesGrabberCORE
|
|||||||
connection.Open();
|
connection.Open();
|
||||||
string sql = "";
|
string sql = "";
|
||||||
if (reblogsOnly)
|
if (reblogsOnly)
|
||||||
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
||||||
else
|
else
|
||||||
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
||||||
|
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
@@ -1159,9 +1248,9 @@ namespace URLNotesGrabberCORE
|
|||||||
connection.Open();
|
connection.Open();
|
||||||
string sql = "";
|
string sql = "";
|
||||||
if (reblogsOnly)
|
if (reblogsOnly)
|
||||||
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
||||||
else
|
else
|
||||||
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
||||||
|
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
@@ -1193,7 +1282,7 @@ namespace URLNotesGrabberCORE
|
|||||||
DBPath ??= GetDefaultDbPath();
|
DBPath ??= GetDefaultDbPath();
|
||||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
connection.Open();
|
connection.Open();
|
||||||
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
|
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath);
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
using (SQLiteDataReader reader = command.ExecuteReader())
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
||||||
@@ -1859,6 +1948,8 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
if (ownsConnection) connection.Open();
|
if (ownsConnection) connection.Open();
|
||||||
|
|
||||||
|
// As in AddPost, IsActive is never written -- neither here nor in the
|
||||||
|
// UPDATE below, which is why an ingest cannot un-remove a post.
|
||||||
string insertSql = @"INSERT INTO Posts (
|
string insertSql = @"INSERT INTO Posts (
|
||||||
BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
|
BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
|
||||||
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
|
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
|
||||||
@@ -1989,7 +2080,7 @@ namespace URLNotesGrabberCORE
|
|||||||
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
||||||
Question, Answer, Title, PostType,
|
Question, Answer, Title, PostType,
|
||||||
HasImage, DateCreated, DateModified
|
HasImage, DateCreated, DateModified
|
||||||
FROM Posts WHERE BlogName = @BlogName";
|
FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath);
|
||||||
using var cmd = new SQLiteCommand(sql, connection);
|
using var cmd = new SQLiteCommand(sql, connection);
|
||||||
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
||||||
using var reader = cmd.ExecuteReader();
|
using var reader = cmd.ExecuteReader();
|
||||||
@@ -2038,7 +2129,7 @@ namespace URLNotesGrabberCORE
|
|||||||
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
||||||
Question, Answer, Title, PostType,
|
Question, Answer, Title, PostType,
|
||||||
HasImage, DateCreated, DateModified
|
HasImage, DateCreated, DateModified
|
||||||
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID";
|
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath);
|
||||||
using var cmd = new SQLiteCommand(sql, connection);
|
using var cmd = new SQLiteCommand(sql, connection);
|
||||||
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
||||||
cmd.Parameters.AddWithValue("@PostID", postId);
|
cmd.Parameters.AddWithValue("@PostID", postId);
|
||||||
@@ -2088,7 +2179,7 @@ namespace URLNotesGrabberCORE
|
|||||||
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
||||||
Question, Answer, Title, PostType,
|
Question, Answer, Title, PostType,
|
||||||
HasImage, DateCreated, DateModified
|
HasImage, DateCreated, DateModified
|
||||||
FROM Posts WHERE PostID = @PostID LIMIT 1";
|
FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1";
|
||||||
using var cmd = new SQLiteCommand(sql, connection);
|
using var cmd = new SQLiteCommand(sql, connection);
|
||||||
cmd.Parameters.AddWithValue("@PostID", postId);
|
cmd.Parameters.AddWithValue("@PostID", postId);
|
||||||
using var reader = cmd.ExecuteReader();
|
using var reader = cmd.ExecuteReader();
|
||||||
|
|||||||
@@ -262,6 +262,47 @@ under its Posts and Notes pages. Removing a blog hides the blog, not what it col
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## `Posts.IsActive` and `Notes.IsActive` — optional, and not in this database yet
|
||||||
|
|
||||||
|
The same flag is being extended to the two content tables, with the same meaning: `0` is
|
||||||
|
removed, anything else — including `NULL` — is live. **Neither column exists in the live
|
||||||
|
`TL.db` as of 2026-07-29**; the DDL quoted above for `Posts` and `Notes` is complete. Like
|
||||||
|
`Blogs.IsActive`, they are written from outside this crawler.
|
||||||
|
|
||||||
|
The crawler therefore treats both as optional, and as nothing it owns:
|
||||||
|
|
||||||
|
- **It never writes them.** No `INSERT` column list names `IsActive`, no `UPDATE` sets it,
|
||||||
|
and `MapPrefixToColumn` — the only place a column name is chosen at runtime — cannot map
|
||||||
|
to it. Re-crawling a removed post or note refreshes its content and leaves the flag at
|
||||||
|
`0`. There is no `INSERT OR REPLACE` on `Posts` or `Notes` for a default to be reset by.
|
||||||
|
- **It filters on them only when they exist.** `HasIsActiveColumn` in `DataAccess.cs` asks
|
||||||
|
`PRAGMA table_info` once per table per database path and caches the answer; the filter
|
||||||
|
is `COALESCE(IsActive, 1) = 1`, and it is dropped entirely when the column is absent.
|
||||||
|
Naming a missing column is a hard SQLite error, so this is what lets one build run
|
||||||
|
against databases on both sides of the change. The cache lives for the process — adding
|
||||||
|
the columns to a live database takes effect on the next run.
|
||||||
|
|
||||||
|
Every read that selects posts or notes carries the filter: `GetPosts`, `GetReplies`,
|
||||||
|
`GetRepliesWithMissingText`, `GetRepliesWithFilledText`, `GetAllPostTextColumns`,
|
||||||
|
`GetAllPostsForBlog`, `GetPost`, `GetPostByIdAnyBlog`, and the engagement queries that
|
||||||
|
count or join `Notes` (`GetBlogs`, `GetBlogsAll`, `GetBlogsForLikes`). The one deliberate
|
||||||
|
omission is the `LEFT JOIN Notes` in `GetPosts`: nothing is selected from it and it can
|
||||||
|
neither add nor remove a row, so filtering it would buy nothing.
|
||||||
|
|
||||||
|
`LegacyPostsDbImporter` is unfiltered too — it reads a foreign legacy database whose
|
||||||
|
`Posts` table is not this schema.
|
||||||
|
|
||||||
|
Two consequences worth stating plainly, both inherited from how `Blogs.IsActive` is
|
||||||
|
handled:
|
||||||
|
|
||||||
|
1. **Removal hides a row; it does not freeze it.** The write paths are keyed on a post the
|
||||||
|
caller already selected, so an ingest or a correction run still overwrites the content
|
||||||
|
of a removed post. Only selection is filtered.
|
||||||
|
2. **`NULL` is live.** Write `0` or `1`, not `NULL`, but a `NULL` leaves the row visible
|
||||||
|
rather than stranding it.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Reproducing the numbers
|
## Reproducing the numbers
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
|
|||||||
@@ -142,6 +142,9 @@ ORDER BY et.tbl;
|
|||||||
-- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
|
-- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
|
||||||
-- list above. Informational only -- e.g. a NEWER backup, or a column this
|
-- list above. Informational only -- e.g. a NEWER backup, or a column this
|
||||||
-- script's expected-list hasn't been updated for. Not an error by itself.
|
-- script's expected-list hasn't been updated for. Not an error by itself.
|
||||||
|
-- Posts.IsActive and Notes.IsActive are listed here and NOT in 1a on
|
||||||
|
-- purpose: they are written by other tools, the app only reads them when
|
||||||
|
-- present, and it must not be told to add them. See TL.db.md.
|
||||||
WITH expected(tbl, col) AS (
|
WITH expected(tbl, col) AS (
|
||||||
VALUES
|
VALUES
|
||||||
('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
|
('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
|
||||||
@@ -150,14 +153,14 @@ WITH expected(tbl, col) AS (
|
|||||||
('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
|
('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
|
||||||
('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
|
('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
|
||||||
('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
|
('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
|
||||||
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
|
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),('Posts','IsActive'),
|
||||||
('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
|
('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
|
||||||
('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
|
('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
|
||||||
('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
|
('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
|
||||||
('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
|
('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
|
||||||
('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
|
('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
|
||||||
('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
|
('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
|
||||||
('Notes','replyText'),
|
('Notes','replyText'),('Notes','IsActive'),
|
||||||
('DailyAPICount','Date'),('DailyAPICount','APICount'),
|
('DailyAPICount','Date'),('DailyAPICount','APICount'),
|
||||||
('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
|
('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
|
||||||
('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')
|
('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')
|
||||||
|
|||||||
Reference in New Issue
Block a user