From 0ff80a0fd3de8583db087aa47fe031cc78a8183a Mon Sep 17 00:00:00 2001 From: jim Date: Tue, 30 Jun 2026 20:41:51 -0500 Subject: [PATCH 1/6] fix: parameterize AddPost fallback UPDATE, guard args indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Posts.hasImage/DateModified fallback update built its WHERE clause via raw string concatenation of blogName/postID, unlike every other query in this method — a blog name containing a single quote would break or inject into the query. Switch it to parameters. --parse, --blogsO, and --bop indexed args[1..3] before checking args.Length, so a missing argument threw IndexOutOfRangeException instead of hitting the intended usage message. --- URLNotesGrabberCORE/DataAccess.cs | 8 ++++++-- URLNotesGrabberCORE/Program.cs | 12 ++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 99e9490..c54d461 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -518,8 +518,12 @@ namespace URLNotesGrabberCORE { if (ownsConnection) connection.Open(); - string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; - SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); + string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID"; + using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); + updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0); + updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + updateCommand.Parameters.AddWithValue("@blogName", blogName); + updateCommand.Parameters.AddWithValue("@postID", postID); updateCommand.ExecuteNonQuery(); } diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 1604dd8..4fe250d 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -175,6 +175,12 @@ namespace URLNotesGrabberCORE break; case "--parse": + if (args.Length < 2) + { + Console.WriteLine("Usage: --parse "); + exitCode = 2; + break; + } string blogNameToParse = args[1]; int postsAdded = 0; try @@ -283,7 +289,7 @@ namespace URLNotesGrabberCORE case "--blogsO": //collect notes from all posts int from = 1, to = 999999, top = 100; - if (args[1] is not null && args[2] is not null && args[3] is not null) + if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null) { from = int.Parse(args[1]); to = int.Parse(args[2]); @@ -293,6 +299,7 @@ namespace URLNotesGrabberCORE { Console.WriteLine("--Expected FROM TO--"); exitCode = 2; + break; } WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), false, from, to, top); break; @@ -300,7 +307,7 @@ namespace URLNotesGrabberCORE case "--bop": //collect notes from all posts from = 1; to = 999999; top = 100; - if (args[1] is not null && args[2] is not null && args[3] is not null) + if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null) { from = int.Parse(args[1]); to = int.Parse(args[2]); @@ -310,6 +317,7 @@ namespace URLNotesGrabberCORE { Console.WriteLine("--Expected FROM TO--"); exitCode = 2; + break; } WriteBlogsToFileAll(settings.GetValue("PathOutputBlogs"), false, from, to, top); break; From f549f020e1ec191dc131b0108063be172fb40949 Mon Sep 17 00:00:00 2001 From: jim Date: Tue, 30 Jun 2026 20:54:18 -0500 Subject: [PATCH 2/6] refactor: standardize SQLiteConnection disposal via using; guard config Replace the try/finally { connection.Close(); } pattern used across most of DataAccess.cs with using declarations, so disposal happens automatically and can't be skipped by a future edit that adds an early return before the finally. Left the shared-connection (ownsConnection) call sites alone since those intentionally outlive a single method call. Also drop a stray unused `using static ... JSType` import, and make a missing ContainsList config setting fail with a clear InvalidOperationException instead of a NullReferenceException from Split(',') on null. --- URLNotesGrabberCORE/DataAccess.cs | 185 +++++++----------------------- URLNotesGrabberCORE/Program.cs | 6 +- 2 files changed, 44 insertions(+), 147 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index c54d461..3241656 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -139,7 +139,7 @@ namespace URLNotesGrabberCORE if (_savedImportPragmas != null) return; - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -175,10 +175,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}"); } - finally - { - connection.Close(); - } } } @@ -190,7 +186,7 @@ namespace URLNotesGrabberCORE if (_savedImportPragmas == null) return; - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -217,7 +213,6 @@ namespace URLNotesGrabberCORE finally { _savedImportPragmas = null; - connection.Close(); } } } @@ -252,7 +247,7 @@ namespace URLNotesGrabberCORE public static void EnsureReplyTextColumnExists(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -296,16 +291,12 @@ namespace URLNotesGrabberCORE // Breakpoint here Console.WriteLine($"Error checking/creating replyText column: {ex.Message}"); } - finally - { - connection.Close(); - } } public static void EnsureBlogsLikesColumnsExist(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -391,10 +382,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}"); } - finally - { - connection.Close(); - } } #region Adds @@ -569,7 +556,7 @@ namespace URLNotesGrabberCORE public static void AddAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -590,10 +577,6 @@ namespace URLNotesGrabberCORE // Breakpoint here //Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null) @@ -602,7 +585,7 @@ namespace URLNotesGrabberCORE //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { } - SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); try { @@ -661,10 +644,6 @@ namespace URLNotesGrabberCORE Console.WriteLine("^^^^^ - SHORTCUT"); } } - finally - { - connection2.Close(); - } return false; } #endregion Adds @@ -680,7 +659,7 @@ namespace URLNotesGrabberCORE public static List> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -858,17 +837,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return posts; } public static List> GetReplies(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -902,17 +877,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return posts; } public static List> GetRepliesWithMissingText(string? DBPath = null, int limit = 50) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -953,17 +924,13 @@ namespace URLNotesGrabberCORE // Breakpoint here Console.WriteLine($"Error getting replies with missing text: {ex.Message}"); } - finally - { - connection.Close(); - } return posts; } public static List> GetRepliesWithFilledText(string? DBPath = null, int? limit = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -1015,17 +982,13 @@ namespace URLNotesGrabberCORE // Breakpoint here Console.WriteLine($"Error getting replies with filled text: {ex.Message}"); } - finally - { - connection.Close(); - } return posts; } public static int GetAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int count = 0; try { AddAPICount(); } catch { } @@ -1054,17 +1017,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return count; } public static List> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> blogs = new List>(); try @@ -1140,17 +1099,13 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error fetching blogs for likes: {ex.Message}"); } - finally - { - connection.Close(); - } return blogs; } public static List GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); try @@ -1185,17 +1140,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return blogs; } public static List GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); try @@ -1230,51 +1181,39 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return blogs; } public static IEnumerable> GetAllPostTextColumns(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); - try + string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { - string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; - using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + using (SQLiteDataReader reader = command.ExecuteReader()) { - using (SQLiteDataReader reader = command.ExecuteReader()) + while (reader.Read()) { - while (reader.Read()) + List rowTexts = new List(); + for (int i = 0; i < reader.FieldCount; i++) { - List rowTexts = new List(); - for (int i = 0; i < reader.FieldCount; i++) + if (!reader.IsDBNull(i)) { - if (!reader.IsDBNull(i)) + var val = reader.GetValue(i); + if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".") { - var val = reader.GetValue(i); - if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".") - { - rowTexts.Add(str); - } + rowTexts.Add(str); } } - if (rowTexts.Count > 0) - { - yield return rowTexts; - } + } + if (rowTexts.Count > 0) + { + yield return rowTexts; } } } } - finally - { - connection.Close(); - connection.Dispose(); - } } #endregion Gets @@ -1284,7 +1223,7 @@ namespace URLNotesGrabberCORE public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1307,16 +1246,12 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1340,10 +1275,6 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } // ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run ----- @@ -1455,7 +1386,7 @@ namespace URLNotesGrabberCORE Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1482,10 +1413,6 @@ namespace URLNotesGrabberCORE return true; } } - finally - { - connection.Close(); - } return false; } @@ -1601,7 +1528,7 @@ namespace URLNotesGrabberCORE public static void UpdateBlogOutput(string blogName, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1621,16 +1548,12 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -1648,10 +1571,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error updating blog likes status: {ex.Message}"); } - finally - { - connection.Close(); - } } // Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we @@ -1661,7 +1580,7 @@ namespace URLNotesGrabberCORE { if (newestTimestamp <= 0) return; DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -1681,10 +1600,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}"); } - finally - { - connection.Close(); - } } // Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the @@ -1692,7 +1607,7 @@ namespace URLNotesGrabberCORE public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -1715,16 +1630,12 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}"); } - finally - { - connection.Close(); - } } public static int UpdateAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int APICount = DataAccess.GetAPICount(); APICount++; @@ -1747,10 +1658,6 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return APICount; } @@ -1758,7 +1665,7 @@ namespace URLNotesGrabberCORE public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int rowsAffected = 0; // A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel. @@ -1801,17 +1708,13 @@ namespace URLNotesGrabberCORE Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}"); Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}"); } - finally - { - connection.Close(); - } return rowsAffected; } public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1847,10 +1750,6 @@ namespace URLNotesGrabberCORE Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}"); return 0; } - finally - { - connection.Close(); - } } #endregion Updates @@ -1861,7 +1760,7 @@ namespace URLNotesGrabberCORE public static void EnsureTTFileHelperColumnsExist(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1907,10 +1806,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}"); } - finally - { - connection.Close(); - } } // INSERT-or-UPDATE for a post arriving from a Tumblr text-file export. diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 4fe250d..07f3eb7 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration; using System.Configuration; using System.Threading; using Microsoft.Extensions.Diagnostics.Latency; -using static System.Runtime.InteropServices.JavaScript.JSType; using System.Text.RegularExpressions; namespace URLNotesGrabberCORE @@ -140,7 +139,10 @@ namespace URLNotesGrabberCORE Console.SetOut(dualLogger); } - List contains = settings.GetValue("ContainsList").Split(',').ToList(); + string? containsListSetting = settings.GetValue("ContainsList"); + if (string.IsNullOrEmpty(containsListSetting)) + throw new InvalidOperationException("ContainsList is not configured in appsettings.json"); + List contains = containsListSetting.Split(',').ToList(); bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false); if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB From f541ec4260fe94df52bd4f3028b375a1e5d7621e Mon Sep 17 00:00:00 2001 From: jim Date: Tue, 30 Jun 2026 21:22:12 -0500 Subject: [PATCH 3/6] fix: correctly detect rate-limit state for single-key API pools IsAllRateLimited() short-circuited true for any pool with 0 or 1 keys, with minRetrySeconds left at 0 regardless of whether that key was actually rate-limited. SleepUntilAnyAvailable() checks "minRetry <= 0" to decide whether to skip sleeping, so with exactly one key it always skipped the wait and let callers hammer the API again immediately after a 429, even mid-cooldown. The per-key loop already computes this correctly for any key count; the special case only needs to cover the true no-keys edge case, where there's nothing to wait on. --- URLNotesGrabberCORE/DataAccess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 3241656..44f695c 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -2626,7 +2626,7 @@ namespace URLNotesGrabberCORE { minRetrySeconds = 0; if (!_usePool || _overrideKey != null) return false; - if (_keys.Count <= 1) return true; + if (_keys.Count == 0) return false; var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); bool all = true; From 33839930e8a24778455b8b448bdb0b9c8c369861 Mon Sep 17 00:00:00 2001 From: jim Date: Thu, 16 Jul 2026 10:20:50 -0500 Subject: [PATCH 4/6] Parse Reblog root url in default .txt ingest mode TraverseDirectory (the no-args ingest path) never read the "Reblog root url:" line, so RootURL stayed unset even though AddPost/UpdatePost already support it via the API-based --likes flow. New scraper output now includes this field; wire it through both AddPost call sites. --- URLNotesGrabberCORE/DataAccess.cs | 2 ++ URLNotesGrabberCORE/Program.cs | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 99e9490..b288d05 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -37,6 +37,7 @@ namespace URLNotesGrabberCORE public string reblogKey; public string reblogName; public string reblogURL; + public string rootURL; public string slug; public string summary; public string tags; @@ -60,6 +61,7 @@ namespace URLNotesGrabberCORE reblogKey = "."; reblogName = "."; reblogURL = "."; + rootURL = "."; slug = "."; summary = "."; tags = "."; diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 1604dd8..3367d41 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -1410,7 +1410,7 @@ if (shouldInsert) DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, - reblog.title, false); + reblog.title, false, rootURL: reblog.rootURL); recordImportStopwatch.Stop(); postsAdded++; @@ -1435,6 +1435,10 @@ if (shouldInsert) { reblog.reblogName = line.Substring(13).Trim(); } + if (line.StartsWith(@"Reblog root url:", StringComparison.OrdinalIgnoreCase)) + { + reblog.rootURL = line.Substring(16).Trim(); + } if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) { reblog.downloadedFiles = line.Substring(17).Trim(); @@ -1538,7 +1542,7 @@ if (shouldInsert) DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, - reblog.title, true); + reblog.title, true, rootURL: reblog.rootURL); recordImportStopwatch.Stop(); postsAdded++; From 21a55250941c7e6c70b3cc279690a32a23a80574 Mon Sep 17 00:00:00 2001 From: jim Date: Thu, 16 Jul 2026 10:23:16 -0500 Subject: [PATCH 5/6] Capture multi-line Body/Downloaded files in default .txt ingest mode TraverseDirectory only ever read the single line immediately after "Body:"/"Downloaded files:", silently dropping every continuation line (multi-paragraph HTML bodies, multiple downloaded filenames). Switch to an indexed line scan so those two fields collect lines until the next recognized field prefix, matching how IngestMode.cs already handles multi-line values. --- URLNotesGrabberCORE/Program.cs | 43 +++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 3367d41..99343b0 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -1352,6 +1352,25 @@ if (shouldInsert) } + // Field prefixes TraverseDirectory recognizes as the start of a new record field. + // Used to know where a multi-line Body/Downloaded files value ends. + private static readonly string[] TraverseDirectoryFieldPrefixes = new[] + { + "Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:", + "Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:", + "Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:", + "Tags:", "Title:" + }; + + private static bool IsTraverseDirectoryFieldLine(string line) + { + foreach (var prefix in TraverseDirectoryFieldPrefixes) + { + if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + static void TraverseDirectory(string path, string outPath, List contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false) { DateTime directoryStart = DateTime.Now; @@ -1388,8 +1407,10 @@ if (shouldInsert) var urls = new List(); var reblog = new ReblogRecord(); - foreach (string line in File.ReadLines(file)) + string[] fileLines = File.ReadAllLines(file); + for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++) { + string line = fileLines[lineIndex]; if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) { if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") @@ -1441,7 +1462,15 @@ if (shouldInsert) } if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) { - reblog.downloadedFiles = line.Substring(17).Trim(); + var valueLines = new List { line.Substring(17).Trim() }; + int nextLineIndex = lineIndex + 1; + while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex])) + { + valueLines.Add(fileLines[nextLineIndex]); + nextLineIndex++; + } + reblog.downloadedFiles = string.Join("\n", valueLines).Trim(); + lineIndex = nextLineIndex - 1; } if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase)) { @@ -1453,7 +1482,15 @@ if (shouldInsert) } if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase)) { - reblog.body = line.Substring(6).Trim(); + var valueLines = new List { line.Substring(6).Trim() }; + int nextLineIndex = lineIndex + 1; + while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex])) + { + valueLines.Add(fileLines[nextLineIndex]); + nextLineIndex++; + } + reblog.body = string.Join("\n", valueLines).Trim(); + lineIndex = nextLineIndex - 1; } if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase)) { From 16147b273e5be25d02d6cc447c6d138531e9b767 Mon Sep 17 00:00:00 2001 From: jim Date: Thu, 16 Jul 2026 10:43:16 -0500 Subject: [PATCH 6/6] docs: document default-mode .txt ingest field parsing in AGENTS.md Note TraverseDirectory's recognized field prefixes, multi-line Body/Downloaded files continuation, and how RootURL now gets populated from both the .txt Reblog root url line and the API-based --likes flow. --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 1ba4253..648e5f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ - `ResponseNotes.cs`: Tumblr API response models - Round-robin API key rotation with rate-limit tracking - Automatic console color assignment per API key for output differentiation +- No-argument mode (`Program.TraverseDirectory`) ingests `.txt` blog export files into `Posts` via `DataAccess.AddPost`. Recognized field prefixes live in `TraverseDirectoryFieldPrefixes`; `Body:` and `Downloaded files:` collect every following line up to the next recognized prefix (multi-line values). `RootURL` is populated from a `Reblog root url:` line the same way it's populated from the API-based `--likes` flow — both paths converge on `DataAccess.AddPost`'s `rootURL` parameter, which `UpdatePost` only overwrites when the incoming value is non-empty (existing `RootURL` is preserved otherwise) ## Developer Guidelines