jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into claude/jovial-mayer-77c2b5

Browse Source
This commit is contained in:
jim
2026-07-29 16:26:50 -05:00
parent 60912c882d 8a4ab2402d
commit 3c85a05afc
4 changed files with 175 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+109 -18
View File
@@ -124,6 +124,91 @@ namespace URLNotesGrabberCORE
return _cachedDbPath;
}
#region IsActive
// Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does:
// 0 = removed elsewhere (Rolodex), anything else (including NULL) = live.
//
// This crawler is a reader of all three. It never writes any IsActive column --
// no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it --
// so a row removed in Rolodex is never resurrected by a re-crawl.
//
// Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added
// from outside this app and are absent from databases that predate them. Naming a
// missing column is a hard SQLite error ("no such column"), so every read asks the
// schema first and simply drops the filter when the column is not there. The answer
// is cached per database path, so adding the columns to a live database takes effect
// on the next run.
private static readonly Dictionary<string, bool> _isActiveColumnCache =
new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
private static readonly object _isActiveColumnLock = new object();
private static bool HasIsActiveColumn(string table, string? DBPath)
{
DBPath ??= GetDefaultDbPath();
string cacheKey = DBPath + "|" + table;
lock (_isActiveColumnLock)
{
if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached))
return cached;
}
bool exists = false;
try
{
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection);
using SQLiteDataReader reader = command.ExecuteReader();
while (reader.Read())
{
if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase))
{
exists = true;
break;
}
}
}
catch (Exception ex)
{
// Breakpoint here
// An unreadable schema is treated as "no column" so the caller's query still runs.
Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}");
}
lock (_isActiveColumnLock)
{
_isActiveColumnCache[cacheKey] = exists;
}
return exists;
}
/// <summary>
/// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it
/// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive.
/// </summary>
private static string AndIsActive(string table, string alias = "", string? DBPath = null)
{
if (!HasIsActiveColumn(table, DBPath)) return string.Empty;
string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + ".";
return $" AND COALESCE({qualifier}IsActive, 1) = 1";
}
/// <summary>
/// Same filter as <see cref="AndIsActive"/>, for a query that has no WHERE clause yet.
/// </summary>
private static string WhereIsActive(string table, string alias = "", string? DBPath = null)
{
string clause = AndIsActive(table, alias, DBPath);
return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length);
}
#endregion IsActive
public static string Q(string input)
{
return "'" + input.Replace("'", "''") + "'";
@@ -464,6 +549,8 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
// IsActive is deliberately absent from this column list: a post removed
// elsewhere must stay removed, so the crawler never writes that flag.
string sql = @"INSERT INTO Posts (
BlogName,
PostID,
@@ -616,6 +703,8 @@ namespace URLNotesGrabberCORE
{
connection2.Open();
// INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note
// that was removed elsewhere leaves the existing row -- and its flag -- alone.
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
{
@@ -713,7 +802,7 @@ namespace URLNotesGrabberCORE
" P.HasNotesGathered," + Environment.NewLine +
" P.NotFound," + Environment.NewLine +
" P.PostDate" + Environment.NewLine +
" FROM Posts P" + Environment.NewLine +
" FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
")," + Environment.NewLine +
"Unioned AS" + Environment.NewLine +
"(" + Environment.NewLine +
@@ -765,8 +854,8 @@ namespace URLNotesGrabberCORE
" LEFT OUTER JOIN " + Environment.NewLine +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine;
" ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
if (beforeDate.HasValue)
{
@@ -875,7 +964,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply'" + AndIsActive("Notes", "Notes", DBPath) + " order by RootBlogName, PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -918,8 +1007,8 @@ namespace URLNotesGrabberCORE
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
MAX(Notes.timestamp) as LatestTimestamp
FROM Notes
WHERE Notes.type = 'reply'
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
WHERE Notes.type = 'reply'
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')" + AndIsActive("Notes", "Notes", DBPath) + @"
GROUP BY Notes.RootBlogName, Notes.PostID
ORDER BY LatestTimestamp ASC
LIMIT @limit";
@@ -966,8 +1055,8 @@ namespace URLNotesGrabberCORE
FROM Posts P
INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
WHERE P.NotFound = 0
AND N.type = 'reply'
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')
AND N.type = 'reply'
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @"
GROUP BY P.BlogName, P.PostID
ORDER BY LatestTimestamp ASC";
@@ -1086,7 +1175,7 @@ namespace URLNotesGrabberCORE
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
AND B.IsActive = 1
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
GROUP BY B.BlogName
ORDER BY MIN(N.Timestamp);";
}
@@ -1101,7 +1190,7 @@ namespace URLNotesGrabberCORE
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
AND B.IsActive = 1
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
AND (
B.LikesPulled = 0
OR COALESCE(B.LikesLastRefreshed, 0)
@@ -1150,9 +1239,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
else
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1191,9 +1280,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
else
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive" + AndIsActive("Notes", "notes", DBPath) + " AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1225,7 +1314,7 @@ namespace URLNotesGrabberCORE
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
@@ -1895,6 +1984,8 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
// As in AddPost, IsActive is never written -- neither here nor in the
// UPDATE below, which is why an ingest cannot un-remove a post.
string insertSql = @"INSERT INTO Posts (
BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
@@ -2025,7 +2116,7 @@ namespace URLNotesGrabberCORE
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE BlogName = @BlogName";
FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath);
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@BlogName", blogName);
using var reader = cmd.ExecuteReader();
@@ -2074,7 +2165,7 @@ namespace URLNotesGrabberCORE
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID";
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath);
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@BlogName", blogName);
cmd.Parameters.AddWithValue("@PostID", postId);
@@ -2124,7 +2215,7 @@ namespace URLNotesGrabberCORE
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE PostID = @PostID LIMIT 1";
FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1";
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@PostID", postId);
using var reader = cmd.ExecuteReader();