jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve import speed, add URL dump, and CLI options

Browse Source 
- Add SQLite import mode pragmas for faster bulk inserts
- Implement -urldump command to extract all URLs from posts
- Add -start [blogname] CLI option for partial traversal
- Support toggling file logging and record import logging via config
- Only update DB rows if values change to reduce writes
- Add DateCreated fields to relevant tables
- Enhance logging and output formatting
- Refactor directory traversal for better control and reporting
This commit is contained in:
jim
2026-04-15 15:14:43 -05:00
parent 390e1284cb
commit 059c9cd527
4 changed files with 581 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+193 -20
View File
@@ -71,6 +71,8 @@ namespace URLNotesGrabberCORE
{
private static IConfiguration? _configuration;
private static HashSet<long>? _postIdsToExclude;
private static readonly object _importPragmaLock = new object();
private static Tuple<string, string>? _savedImportPragmas;
static DataAccess()
{
@@ -106,6 +108,95 @@ namespace URLNotesGrabberCORE
return dateTime;
}
public static void EnableImportModePragmas(string DBPath = @"TL.db")
{
lock (_importPragmaLock)
{
if (_savedImportPragmas != null)
return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string currentJournalMode = "delete";
string currentSynchronous = "2";
using (var cmd = new SQLiteCommand("PRAGMA journal_mode;", connection))
{
currentJournalMode = Convert.ToString(cmd.ExecuteScalar()) ?? "delete";
}
using (var cmd = new SQLiteCommand("PRAGMA synchronous;", connection))
{
currentSynchronous = Convert.ToString(cmd.ExecuteScalar()) ?? "2";
}
_savedImportPragmas = new Tuple<string, string>(currentJournalMode, currentSynchronous);
using (var cmd = new SQLiteCommand("PRAGMA journal_mode=WAL;", connection))
{
cmd.ExecuteNonQuery();
}
using (var cmd = new SQLiteCommand("PRAGMA synchronous=NORMAL;", connection))
{
cmd.ExecuteNonQuery();
}
Console.WriteLine($"[SQLite Import Mode] Enabled | journal_mode=WAL | synchronous=NORMAL (previous: journal_mode={currentJournalMode}, synchronous={currentSynchronous})");
}
catch (Exception ex)
{
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
}
finally
{
connection.Close();
}
}
}
public static void RestoreImportModePragmas(string DBPath = @"TL.db")
{
lock (_importPragmaLock)
{
if (_savedImportPragmas == null)
return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string priorJournalMode = _savedImportPragmas.Item1;
string priorSynchronous = _savedImportPragmas.Item2;
using (var cmd = new SQLiteCommand($"PRAGMA journal_mode={priorJournalMode};", connection))
{
cmd.ExecuteNonQuery();
}
using (var cmd = new SQLiteCommand($"PRAGMA synchronous={priorSynchronous};", connection))
{
cmd.ExecuteNonQuery();
}
Console.WriteLine($"[SQLite Import Mode] Restored | journal_mode={priorJournalMode} | synchronous={priorSynchronous}");
}
catch (Exception ex)
{
Console.WriteLine($"[SQLite Import Mode] Failed to restore pragmas: {ex.Message}");
}
finally
{
_savedImportPragmas = null;
connection.Close();
}
}
}
public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -220,12 +311,13 @@ namespace URLNotesGrabberCORE
connection.Open();
// Insert BlogName and DateAdded (current UTC datetime)
string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @ByLikes)";
string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, DateCreated, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @DateCreated, @ByLikes)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@BlogName", blogName);
command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@ByLikes", byLikes ? 1 : 0);
command.ExecuteNonQuery();
}
@@ -277,12 +369,13 @@ namespace URLNotesGrabberCORE
Answer,
Title,
DateModified,
DateCreated,
RootBlogName,
RootURL,
HasImage,
ByLikes
) VALUES (" +
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection);
int rowsInserted = 0;
@@ -301,7 +394,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string updateSql = "UPDATE Posts SET hasImage = " + hasImage + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.ExecuteNonQuery();
@@ -355,10 +448,11 @@ namespace URLNotesGrabberCORE
// Use INSERT OR IGNORE to avoid UNIQUE constraint errors when the date row already exists.
// Also explicitly initialize APICount to 0 in case the table has no default.
string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount) values(@date, 0)";
string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount, DateCreated) values(@date, 0, @DateCreated)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
}
@@ -378,15 +472,13 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { }
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection2.Open();
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified)";
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
{
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
@@ -396,9 +488,23 @@ namespace URLNotesGrabberCORE
command.Parameters.AddWithValue("@Type", type ?? string.Empty);
command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
int rowsInserted = command.ExecuteNonQuery();
if (rowsInserted == 1)
{
ConsoleColor previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
Console.ForegroundColor = previousColor;
//Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
}
else
{
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
}
// Only update HasBeenOutput if a new note was inserted
if (rowsInserted == 1)
{
@@ -462,12 +568,12 @@ namespace URLNotesGrabberCORE
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine;
"WHERE NotFound = 0 " + Environment.NewLine;
if (withoutNotesOnly)
{
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
sql += "OR ( NotFound = 0 AND Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
}
// Filter by NotesGatheredDateTime if beforeDate is provided
@@ -885,6 +991,45 @@ namespace URLNotesGrabberCORE
}
return blogs;
}
public static IEnumerable<List<string>> GetAllPostTextColumns(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
try
{
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
List<string> rowTexts = new List<string>();
for (int i = 0; i < reader.FieldCount; i++)
{
if (!reader.IsDBNull(i))
{
var val = reader.GetValue(i);
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
{
rowTexts.Add(str);
}
}
}
if (rowTexts.Count > 0)
{
yield return rowTexts;
}
}
}
}
}
finally
{
connection.Close();
connection.Dispose();
}
}
#endregion Gets
#region Updates
@@ -899,7 +1044,7 @@ namespace URLNotesGrabberCORE
connection.Open();
//string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID";
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
@@ -932,7 +1077,7 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID";
string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(NotFound, 0) <> 1";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
@@ -961,7 +1106,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID";
string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(postDate, '') <> @postDate";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty);
@@ -996,7 +1141,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID";
string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID AND IFNULL(timestamp, 0) <> @timestamp";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@timestamp", timestamp);
@@ -1056,7 +1201,30 @@ namespace URLNotesGrabberCORE
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
sql += "hasImage = @hasImage, ";
sql += "ByLikes = @byLikes ";
sql += " WHERE BlogName = @BlogName AND PostID = @PostID";
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
sql += "IFNULL(postDate, '') <> @postDate OR ";
sql += "IFNULL(reblogURL, '') <> @reblogURL OR ";
sql += "IFNULL(postURL, '') <> @postURL OR ";
sql += "IFNULL(slug, '') <> @slug OR ";
sql += "IFNULL(reblogKey, '') <> @reblogKey OR ";
sql += "IFNULL(reblogName, '') <> @reblogName OR ";
sql += "IFNULL(summary, '') <> @summary OR ";
sql += "IFNULL(quote, '') <> @quote OR ";
sql += "IFNULL(body, '') <> @body OR ";
sql += "IFNULL(tags, '') <> @tags OR ";
sql += "IFNULL(link, '') <> @link OR ";
sql += "IFNULL(photoURL, '') <> @photoURL OR ";
sql += "IFNULL(photoCaption, '') <> @photoCaption OR ";
sql += "IFNULL(downloadedFiles, '') <> @downloadedFiles OR ";
sql += "IFNULL(audioCaption, '') <> @audioCaption OR ";
sql += "IFNULL(question, '') <> @question OR ";
sql += "IFNULL(answer, '') <> @answer OR ";
sql += "IFNULL(title, '') <> @title OR ";
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
sql += "IFNULL(ByLikes, 0) <> @byLikes OR ";
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)";
sql += ")";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1110,7 +1278,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName";
string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName AND IFNULL(HasBeenOutput, 0) <> 1";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@BlogName", blogName);
@@ -1136,7 +1304,7 @@ namespace URLNotesGrabberCORE
try
{
connection.Open();
string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name";
string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name AND (IFNULL(LikesPulled, 0) <> @pulled OR IFNULL(LikesCursor, 0) <> @cursor)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@pulled", likesPulled);
@@ -1198,7 +1366,7 @@ namespace URLNotesGrabberCORE
connection.Open();
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
@@ -1211,7 +1379,7 @@ namespace URLNotesGrabberCORE
if (rowsAffected == 0)
{
Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}");
Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
}
@@ -1241,7 +1409,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'";
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
@@ -1252,7 +1420,7 @@ namespace URLNotesGrabberCORE
if (rowsAffected == 0)
{
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}/{postID} (rows not found or values unchanged)");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
}
@@ -1417,6 +1585,11 @@ namespace URLNotesGrabberCORE
myDeserializedClass = deserializedResult;
myDeserializedClass.rawJson = myJsonResponse;
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
{
myDeserializedClass.statusCode = "NotFound";
}
// If the response JSON indicates a 429 (Too Many Requests) via meta.status or message,
// treat it like a rate-limited response and attempt to read Retry headers.
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;