Improve import speed, add URL dump, and CLI options

- Add SQLite import mode pragmas for faster bulk inserts
- Implement -urldump command to extract all URLs from posts
- Add -start [blogname] CLI option for partial traversal
- Support toggling file logging and record import logging via config
- Only update DB rows if values change to reduce writes
- Add DateCreated fields to relevant tables
- Enhance logging and output formatting
- Refactor directory traversal for better control and reporting

URLNotesGrabberCORE/DataAccess.cs

@@ -71,6 +71,8 @@ namespace URLNotesGrabberCORE
     {
         private static IConfiguration? _configuration;
         private static HashSet<long>? _postIdsToExclude;
+        private static readonly object _importPragmaLock = new object();
+        private static Tuple<string, string>? _savedImportPragmas;
 
         static DataAccess()
         {
@@ -106,6 +108,95 @@ namespace URLNotesGrabberCORE
             return dateTime;
         }
 
+        public static void EnableImportModePragmas(string DBPath = @"TL.db")
+        {
+            lock (_importPragmaLock)
+            {
+                if (_savedImportPragmas != null)
+                    return;
+
+                SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+                try
+                {
+                    connection.Open();
+
+                    string currentJournalMode = "delete";
+                    string currentSynchronous = "2";
+
+                    using (var cmd = new SQLiteCommand("PRAGMA journal_mode;", connection))
+                    {
+                        currentJournalMode = Convert.ToString(cmd.ExecuteScalar()) ?? "delete";
+                    }
+
+                    using (var cmd = new SQLiteCommand("PRAGMA synchronous;", connection))
+                    {
+                        currentSynchronous = Convert.ToString(cmd.ExecuteScalar()) ?? "2";
+                    }
+
+                    _savedImportPragmas = new Tuple<string, string>(currentJournalMode, currentSynchronous);
+
+                    using (var cmd = new SQLiteCommand("PRAGMA journal_mode=WAL;", connection))
+                    {
+                        cmd.ExecuteNonQuery();
+                    }
+
+                    using (var cmd = new SQLiteCommand("PRAGMA synchronous=NORMAL;", connection))
+                    {
+                        cmd.ExecuteNonQuery();
+                    }
+
+                    Console.WriteLine($"[SQLite Import Mode] Enabled | journal_mode=WAL | synchronous=NORMAL (previous: journal_mode={currentJournalMode}, synchronous={currentSynchronous})");
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
+                }
+                finally
+                {
+                    connection.Close();
+                }
+            }
+        }
+
+        public static void RestoreImportModePragmas(string DBPath = @"TL.db")
+        {
+            lock (_importPragmaLock)
+            {
+                if (_savedImportPragmas == null)
+                    return;
+
+                SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+                try
+                {
+                    connection.Open();
+
+                    string priorJournalMode = _savedImportPragmas.Item1;
+                    string priorSynchronous = _savedImportPragmas.Item2;
+
+                    using (var cmd = new SQLiteCommand($"PRAGMA journal_mode={priorJournalMode};", connection))
+                    {
+                        cmd.ExecuteNonQuery();
+                    }
+
+                    using (var cmd = new SQLiteCommand($"PRAGMA synchronous={priorSynchronous};", connection))
+                    {
+                        cmd.ExecuteNonQuery();
+                    }
+
+                    Console.WriteLine($"[SQLite Import Mode] Restored | journal_mode={priorJournalMode} | synchronous={priorSynchronous}");
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine($"[SQLite Import Mode] Failed to restore pragmas: {ex.Message}");
+                }
+                finally
+                {
+                    _savedImportPragmas = null;
+                    connection.Close();
+                }
+            }
+        }
+
         public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -220,12 +311,13 @@ namespace URLNotesGrabberCORE
                 connection.Open();
 
                 // Insert BlogName and DateAdded (current UTC datetime)
-                string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @ByLikes)";
+                string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, DateCreated, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @DateCreated, @ByLikes)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@BlogName", blogName);
                     command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                     command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                     command.Parameters.AddWithValue("@ByLikes", byLikes ? 1 : 0);
                     command.ExecuteNonQuery();
                 }
@@ -277,12 +369,13 @@ namespace URLNotesGrabberCORE
                                 Answer,
                                 Title,
                                 DateModified,
+                                DateCreated,
                                 RootBlogName,
                                 RootURL,
                                 HasImage,
                                 ByLikes
                                 ) VALUES (" +
-                    Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
+                    Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 int rowsInserted = 0;
@@ -301,7 +394,7 @@ namespace URLNotesGrabberCORE
                         {
                             connection.Open();
 
-                            string updateSql = "UPDATE Posts SET hasImage = " + hasImage + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
+                            string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
                             SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
 
                             updateCommand.ExecuteNonQuery();
@@ -355,10 +448,11 @@ namespace URLNotesGrabberCORE
 
                 // Use INSERT OR IGNORE to avoid UNIQUE constraint errors when the date row already exists.
                 // Also explicitly initialize APICount to 0 in case the table has no default.
-                string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount) values(@date, 0)";
+                string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount, DateCreated) values(@date, 0, @DateCreated)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
+                    command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                     command.ExecuteNonQuery();
                 }
             }
@@ -378,15 +472,13 @@ namespace URLNotesGrabberCORE
             //try { AddPost(rootBlogName, postID, DBPath); } catch { }
             try { AddBlog(noteBlogName, false, DBPath); } catch { }
 
-            Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
-
             SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
 
             try
             {
                 connection2.Open();
 
-                string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified)";
+                string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
                 {
                     command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
@@ -396,9 +488,23 @@ namespace URLNotesGrabberCORE
                     command.Parameters.AddWithValue("@Type", type ?? string.Empty);
                     command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                     command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
 
                     int rowsInserted = command.ExecuteNonQuery();
 
+                    if (rowsInserted == 1)
+                    {
+                        ConsoleColor previousColor = Console.ForegroundColor;
+                        Console.ForegroundColor = ConsoleColor.Green;
+                        Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
+                        Console.ForegroundColor = previousColor;
+                        //Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
+                    }
+                    else
+                    {
+                        Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
+                    }
+
                     // Only update HasBeenOutput if a new note was inserted
                     if (rowsInserted == 1)
                     {
@@ -462,12 +568,12 @@ namespace URLNotesGrabberCORE
                              "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
                              "      LEFT OUTER JOIN " + Environment.NewLine +
                              "   (      select BlogName, count(PostID) as CNT from Posts group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
-                             "WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine;
+                             "WHERE NotFound = 0 " + Environment.NewLine;
 
                 if (withoutNotesOnly)
                 {
                     sql += " and HasNotesGathered = 0 " + Environment.NewLine;
-                    sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
+                    sql += "OR ( NotFound = 0 AND Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
                 }
 
                 // Filter by NotesGatheredDateTime if beforeDate is provided
@@ -885,6 +991,45 @@ namespace URLNotesGrabberCORE
             }
             return blogs;
         }
+        public static IEnumerable<List<string>> GetAllPostTextColumns(string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            try
+            {
+                string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            List<string> rowTexts = new List<string>();
+                            for (int i = 0; i < reader.FieldCount; i++)
+                            {
+                                if (!reader.IsDBNull(i))
+                                {
+                                    var val = reader.GetValue(i);
+                                    if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
+                                    {
+                                        rowTexts.Add(str);
+                                    }
+                                }
+                            }
+                            if (rowTexts.Count > 0)
+                            {
+                                yield return rowTexts;
+                            }
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                connection.Close();
+                connection.Dispose();
+            }
+        }
         #endregion Gets
 
         #region Updates
@@ -899,7 +1044,7 @@ namespace URLNotesGrabberCORE
                 connection.Open();
 
                 //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
-                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID";
+                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
@@ -932,7 +1077,7 @@ namespace URLNotesGrabberCORE
 
                 connection.Open();
 
-                string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID";
+                string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(NotFound, 0) <> 1";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
@@ -961,7 +1106,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID";
+                string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(postDate, '') <> @postDate";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty);
@@ -996,7 +1141,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID";
+                string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID AND IFNULL(timestamp, 0) <> @timestamp";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@timestamp", timestamp);
@@ -1056,7 +1201,30 @@ namespace URLNotesGrabberCORE
                 sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
                 sql += "hasImage = @hasImage, ";
                 sql += "ByLikes = @byLikes ";
-                sql += " WHERE BlogName = @BlogName AND PostID = @PostID";
+                sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
+                sql += "IFNULL(postDate, '') <> @postDate OR ";
+                sql += "IFNULL(reblogURL, '') <> @reblogURL OR ";
+                sql += "IFNULL(postURL, '') <> @postURL OR ";
+                sql += "IFNULL(slug, '') <> @slug OR ";
+                sql += "IFNULL(reblogKey, '') <> @reblogKey OR ";
+                sql += "IFNULL(reblogName, '') <> @reblogName OR ";
+                sql += "IFNULL(summary, '') <> @summary OR ";
+                sql += "IFNULL(quote, '') <> @quote OR ";
+                sql += "IFNULL(body, '') <> @body OR ";
+                sql += "IFNULL(tags, '') <> @tags OR ";
+                sql += "IFNULL(link, '') <> @link OR ";
+                sql += "IFNULL(photoURL, '') <> @photoURL OR ";
+                sql += "IFNULL(photoCaption, '') <> @photoCaption OR ";
+                sql += "IFNULL(downloadedFiles, '') <> @downloadedFiles OR ";
+                sql += "IFNULL(audioCaption, '') <> @audioCaption OR ";
+                sql += "IFNULL(question, '') <> @question OR ";
+                sql += "IFNULL(answer, '') <> @answer OR ";
+                sql += "IFNULL(title, '') <> @title OR ";
+                sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
+                sql += "IFNULL(ByLikes, 0) <> @byLikes OR ";
+                sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
+                sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)";
+                sql += ")";
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -1110,7 +1278,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName";
+                string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName AND IFNULL(HasBeenOutput, 0) <> 1";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@BlogName", blogName);
@@ -1136,7 +1304,7 @@ namespace URLNotesGrabberCORE
             try
             {
                 connection.Open();
-                string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name";
+                string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name AND (IFNULL(LikesPulled, 0) <> @pulled OR IFNULL(LikesCursor, 0) <> @cursor)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@pulled", likesPulled);
@@ -1198,7 +1366,7 @@ namespace URLNotesGrabberCORE
                 connection.Open();
 
                 //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
-                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
+                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@replyText", replyText ?? ".");
@@ -1211,7 +1379,7 @@ namespace URLNotesGrabberCORE
                     
                     if (rowsAffected == 0)
                     {
-                        Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}");
+                        Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
                         Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
                         Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
                     }
@@ -1241,7 +1409,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'";
+                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@replyText", replyText ?? ".");
@@ -1252,7 +1420,7 @@ namespace URLNotesGrabberCORE
                     
                     if (rowsAffected == 0)
                     {
-                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}/{postID} (rows not found or values unchanged)");
                         Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
                         Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
                     }
@@ -1417,6 +1585,11 @@ namespace URLNotesGrabberCORE
                     myDeserializedClass = deserializedResult;
                     myDeserializedClass.rawJson = myJsonResponse;
 
+                    if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
+                    {
+                        myDeserializedClass.statusCode = "NotFound";
+                    }
+
                     // If the response JSON indicates a 429 (Too Many Requests) via meta.status or message,
                     // treat it like a rate-limited response and attempt to read Retry headers.
                     bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;

URLNotesGrabberCORE/Program.cs

@@ -6,6 +6,7 @@ using System.Configuration;
 using System.Threading;
 using Microsoft.Extensions.Diagnostics.Latency;
 using static System.Runtime.InteropServices.JavaScript.JSType;
+using System.Text.RegularExpressions;
 
 namespace URLNotesGrabberCORE
 {
@@ -22,6 +23,7 @@ namespace URLNotesGrabberCORE
             var settings = config.GetSection("appSettings");
 
             string apiSectionName = "TumblrApi";
+            string startFromBlogName = string.Empty;
             List<string> filteredArgs = new List<string>();
             for (int i = 0; i < args.Length; i++)
             {
@@ -54,6 +56,21 @@ namespace URLNotesGrabberCORE
                     continue;
                 }
 
+                if (string.Equals(args[i], "-start", StringComparison.OrdinalIgnoreCase) ||
+                    string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
+                {
+                    if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
+                    {
+                        startFromBlogName = args[i + 1].Trim();
+                        i++;
+                    }
+                    else
+                    {
+                        Console.WriteLine("--Missing blog name after -start/--start. Ignoring.--");
+                    }
+                    continue;
+                }
+
                 filteredArgs.Add(args[i]);
             }
 
@@ -61,6 +78,9 @@ namespace URLNotesGrabberCORE
             APIAccess.SetApiConfigSection(apiSectionName);
 
             // Setup Dual Logging
+            bool enableFileLogging = settings.GetValue("EnableFileLogging", true);
+            if (enableFileLogging)
+            {
                 string logPath = "console_output.log";
                 if (File.Exists(logPath))
                 {
@@ -80,13 +100,23 @@ namespace URLNotesGrabberCORE
                 StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
                 DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
                 Console.SetOut(dualLogger);
+            }
 
             List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
+            bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
 
             if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
             {
                 int postsAdded = 0;
-                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded);
+                try
+                {
+                    DataAccess.EnableImportModePragmas();
+                    TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports);
+                }
+                finally
+                {
+                    DataAccess.RestoreImportModePragmas();
+                }
                 Console.WriteLine($"Total posts added: {postsAdded}");
             }
             else
@@ -116,10 +146,14 @@ namespace URLNotesGrabberCORE
 
                         Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param");
 
+                        Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
+
                         Console.WriteLine("-api3\t Use TumblrApi3 settings from appsettings.json");
 
                         Console.WriteLine("-api4\t Use TumblrApi4 settings from appsettings.json");
 
+                        Console.WriteLine("-start [blogname]\t Start traversal alphabetically at this blog name");
+
                         Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
 
                         break;
@@ -127,7 +161,15 @@ namespace URLNotesGrabberCORE
                     case "-parse":
                         string blogNameToParse = args[1];
                         int postsAdded = 0;
-                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse);
+                        try
+                        {
+                            DataAccess.EnableImportModePragmas();
+                            TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse, startFromBlogName, logTraversalRecordImports);
+                        }
+                        finally
+                        {
+                            DataAccess.RestoreImportModePragmas();
+                        }
                         Console.WriteLine($"Total posts added: {postsAdded}");
                         break;
                     
@@ -237,6 +279,10 @@ namespace URLNotesGrabberCORE
                         CollectLikes(likeBlog, contains).GetAwaiter().GetResult();
                         break;
 
+                    case "-urldump":
+                        DumpUrls(settings.GetValue<string>("PathOutputUrls"));
+                        break;
+
                     default:
                         Console.WriteLine("** Unknown Command ** " + args[0]);
                         break;
@@ -320,6 +366,45 @@ namespace URLNotesGrabberCORE
             }
         }
 
+        static void DumpUrls(string? outPath)
+        {
+            if (string.IsNullOrWhiteSpace(outPath))
+            {
+                Console.WriteLine("--PathOutputUrls is not configured in appsettings.json--");
+                return;
+            }
+
+            Console.WriteLine($"Starting URL extraction to {outPath}...");
+            HashSet<string> uniqueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            Regex urlRegex = new Regex(@"https?://[^\s""'<>]+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
+
+            int rowsProcessed = 0;
+            foreach (var texts in DataAccess.GetAllPostTextColumns())
+            {
+                rowsProcessed++;
+                if (rowsProcessed % 10000 == 0)
+                {
+                    Console.WriteLine($"Scanned {rowsProcessed} rows... found {uniqueUrls.Count} unique URLs so far.");
+                }
+
+                foreach (var text in texts)
+                {
+                    var matches = urlRegex.Matches(text);
+                    foreach (Match match in matches)
+                    {
+                        uniqueUrls.Add(match.Value);
+                    }
+                }
+            }
+
+            Console.WriteLine($"Scan complete. Sorting and saving {uniqueUrls.Count} distinct URLs...");
+            var sortedUrls = uniqueUrls.ToList();
+            sortedUrls.Sort();
+
+            File.WriteAllLines(outPath, sortedUrls);
+            Console.WriteLine($"Saved URLs to {outPath}");
+        }
+
         protected static bool ContainsAny(string input, List<string> contains)
         {
             if (string.IsNullOrEmpty(input)) return false;
@@ -335,6 +420,20 @@ namespace URLNotesGrabberCORE
             return false;
         }
 
+        protected static string NormalizeBlogFolderName(string folderName)
+        {
+            return folderName
+                .Replace("_1", "")
+                .Replace("_2", "")
+                .Replace("_3", "")
+                .Replace("_4", "")
+                .Replace("_5", "")
+                .Replace("_6", "")
+                .Replace("_7", "")
+                .Replace("_8", "")
+                .Replace("_9", "");
+        }
+
         static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
         {
             try
@@ -494,6 +593,44 @@ namespace URLNotesGrabberCORE
             }
         }
 
+        static ConsoleColor GetMatchedCounterColor(long matchedCount)
+        {
+            if (matchedCount <= 0)
+                return ConsoleColor.White;
+
+            return ((matchedCount - 1) % 3) switch
+            {
+                0 => ConsoleColor.Red,
+                1 => ConsoleColor.Green,
+                _ => ConsoleColor.Blue
+            };
+        }
+
+        static void WriteLikesTotalsLine(string blogName, string label, long parsedForBlog, long matchedForBlog, int likedCountForBlog)
+        {
+            if (likedCountForBlog > 0)
+            {
+                double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
+                double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
+
+                Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: ");
+                var previousColor = Console.ForegroundColor;
+                Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
+                Console.Write(matchedForBlog);
+                Console.ForegroundColor = previousColor;
+                Console.WriteLine($"/{likedCountForBlog} ({matchedPct:F2}%)");
+            }
+            else
+            {
+                Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog} | Matched: ");
+                var previousColor = Console.ForegroundColor;
+                Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
+                Console.Write(matchedForBlog);
+                Console.ForegroundColor = previousColor;
+                Console.WriteLine();
+            }
+        }
+
         static async Task CollectLikes(string specificBlog, List<string> contains)
         {
             try
@@ -691,16 +828,7 @@ namespace URLNotesGrabberCORE
                             }
                         }
 
-                        if (likedCountForBlog > 0)
+                        WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
-                        {
-                            double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
-                            double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
-                            Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)");
-                        }
-                        else
-                        {
-                            Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}");
-                        }
 
                         // Determine the next BeforeCursor.
                         long nextCursor = 0;
@@ -728,16 +856,7 @@ namespace URLNotesGrabberCORE
                         await Task.Delay(1000); // 1-second delay between pages
                     }
 
-                    if (likedCountForBlog > 0)
+                    WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
-                    {
-                        double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
-                        double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
-                        Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)");
-                    }
-                    else
-                    {
-                        Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}");
-                    }
                 }
 
                 Console.WriteLine("Likes collection complete.");
@@ -753,6 +872,11 @@ namespace URLNotesGrabberCORE
         {
             try
             {
+                static bool IsNotFound(Root? r)
+                {
+                    return (r?.meta != null && r.meta.status == 404) || string.Equals(r?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
+                }
+
                 int APICount = DataAccess.GetAPICount();
                 Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
                 //Thread.Sleep(3000);
@@ -765,9 +889,9 @@ namespace URLNotesGrabberCORE
                 var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
 
                 // Handle 404 and error codes
-                if (response?.meta != null && response.meta.status == 404)
+                if (IsNotFound(response))
                 {
-                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)");
+                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}");
                     DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                     Thread.Sleep(1000);
                     return "NotFound";
@@ -777,12 +901,6 @@ namespace URLNotesGrabberCORE
                     Console.WriteLine("##### Response is null - API Failure? ###");
                     return "FAILURE";
                 }
-                if (response.statusCode == "NotFound")
-                {
-                    Thread.Sleep(1000);
-                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
-                    return response.statusCode;
-                }
                 if (response.statusCode == "TooManyRequests")
                 {
                     for (int s = 0; s <= response.retryInSeconds; s += 60)
@@ -801,11 +919,23 @@ namespace URLNotesGrabberCORE
 
                     if (response?.response == null)
                     {
+                        if (IsNotFound(response))
+                        {
+                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (empty response payload)");
+                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                            return "NotFound";
+                        }
                         Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
                         return "FAILURE";
                     }
                     if (response.response.notes == null)
                     {
+                        if (IsNotFound(response))
+                        {
+                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (notes payload missing)");
+                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                            return "NotFound";
+                        }
                         Console.WriteLine("##### Notes Null - WHY? ###");
                         return "FAILURE";
                     }
@@ -831,6 +961,12 @@ namespace URLNotesGrabberCORE
 
                     // Fetch next page
                     response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
+                    if (IsNotFound(response))
+                    {
+                        Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}");
+                        DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                        return "NotFound";
+                    }
                 }
 
                 Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
@@ -893,15 +1029,14 @@ namespace URLNotesGrabberCORE
                         {
                             DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
                         }
+                        else if (status == "NotFound")
+                        {
+                            Console.WriteLine("GrabNotes Result: NotFound");
+                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                        }
                         else
                         {
                             Console.WriteLine("GrabNotes Result: " + status);
-
-                            //if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue
-                            if (status != "RateLimitExceeded")
-                            {
-                                DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
-                            }
                         }
 
                         // Re-fetch the updated list after processing the current post
@@ -916,7 +1051,13 @@ namespace URLNotesGrabberCORE
         }
 
 
-        static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "")
+        static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
+        {
+            DateTime directoryStart = DateTime.Now;
+            int directoryRecordsImported = 0;
+            Console.WriteLine($"[Directory Start] {path} | {directoryStart:yyyy-MM-dd HH:mm:ss.fff}");
+
+            try
             {
                 // Get all directories in the current directory and sort them alphabetically
                 var directories = Directory.GetDirectories(path);
@@ -925,7 +1066,7 @@ namespace URLNotesGrabberCORE
                 foreach (var directory in directories)
                 {
                     Console.WriteLine("Directory: " + directory);
-                TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories
+                    TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName, startFromBlogName, logRecordImports); // Recursively traverse subdirectories
                 }
 
                 try
@@ -934,7 +1075,12 @@ namespace URLNotesGrabberCORE
                     // Process all files in the current directory
                     foreach (var file in Directory.GetFiles(path))
                     {
-                    if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
+                        string normalizedDirectoryName = NormalizeBlogFolderName(new DirectoryInfo(path).Name);
+                        bool isAtOrAfterStart = string.IsNullOrWhiteSpace(startFromBlogName) || string.Compare(normalizedDirectoryName, startFromBlogName, StringComparison.OrdinalIgnoreCase) >= 0;
+
+                        if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)
+                            && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)
+                            && isAtOrAfterStart)
                         {
                             try
                             {
@@ -956,13 +1102,20 @@ namespace URLNotesGrabberCORE
                                                 {
                                                     headerWasWritten = true;
                                                 }
-                                            string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
+                                                string curDir = NormalizeBlogFolderName(currentDir.Name);
                                                 urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
+
+                                                var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
                                                 DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                     reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                     reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                     reblog.title, false);
+                                                recordImportStopwatch.Stop();
+
                                                 postsAdded++;
+                                                directoryRecordsImported++;
+                                                if (logRecordImports)
+                                                    Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
 
                                                 // Output hyperlink and post date
                                                 //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
@@ -1077,13 +1230,20 @@ namespace URLNotesGrabberCORE
                                             //    Console.WriteLine(reblog.reblogName);
                                             //    Console.WriteLine(reblog.downloadedFiles);
                                             //}
-                                        string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
+                                            string curDir = NormalizeBlogFolderName(currentDir.Name);
                                             urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
+
+                                            var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
                                             DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                     reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                     reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                     reblog.title, true);
+                                            recordImportStopwatch.Stop();
+
                                             postsAdded++;
+                                            directoryRecordsImported++;
+                                            if (logRecordImports)
+                                                Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
 
                                             // Output hyperlink and post date
                                             //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
@@ -1115,6 +1275,13 @@ namespace URLNotesGrabberCORE
                     Console.WriteLine($"An error occurred: {ex.Message}");
                 }
             }
+            finally
+            {
+                DateTime directoryEnd = DateTime.Now;
+                TimeSpan elapsed = directoryEnd - directoryStart;
+                Console.WriteLine($"[Directory End] {path} | {directoryEnd:yyyy-MM-dd HH:mm:ss.fff} | Duration: {elapsed:hh\\:mm\\:ss\\.fff} | Records Imported: {directoryRecordsImported}");
+            }
+        }
 
 
         static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")

URLNotesGrabberCORE/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "URLNotesGrabberCORE": {
       "commandName": "Project",
-      "commandLineArgs": "-likes timothywrite"
+      "commandLineArgs": "-collect 1 -api4"
     }
   }
 }

URLNotesGrabberCORE/appsettings.json

@@ -6,9 +6,12 @@
     "PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt",
     "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
     "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
+    "PathOutputUrls": "u:\\jim\\Documents\\Web Copies\\blogs\\GetUrls.txt",
     "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
     "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot",
-    "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218"
+    "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218",
+    "EnableFileLogging": false,
+    "LogTraversalRecordImports": false
   },
   "TumblrApi": {
     "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",