speed up no-args directory import with shared connection + lazy blog cache

AddBlog/AddPost/UpdatePost/UpdatePostSetDate now reuse a single SQLiteConnection
when an import session is active, instead of opening and closing one per call.
AddBlog also short-circuits on an in-memory HashSet of blog names already
attempted this run. Other entry points are unaffected since they never call
BeginImportSession.

Co-Authored-By: Claude Opus 4.7 <[email protected]>

URLNotesGrabberCORE/DataAccess.cs

@@ -75,6 +75,9 @@ namespace URLNotesGrabberCORE
         private static readonly object _importPragmaLock = new object();
         private static Tuple<string, string>? _savedImportPragmas;
         private static string? _cachedDbPath;
+        private static SQLiteConnection? _importConnection;
+        private static HashSet<string>? _importBlogCache;
+        private static readonly object _importSessionLock = new object();
 
         static DataAccess()
         {
@@ -219,6 +222,33 @@ namespace URLNotesGrabberCORE
             }
         }
 
+        public static void BeginImportSession(string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+            lock (_importSessionLock)
+            {
+                if (_importConnection != null) return;
+
+                var conn = new SQLiteConnection("Data Source=" + DBPath);
+                conn.Open();
+                _importConnection = conn;
+
+                _importBlogCache = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+                Console.WriteLine("[Import Session] Opened single connection | blog cache initialized (lazy)");
+            }
+        }
+
+        public static void EndImportSession()
+        {
+            lock (_importSessionLock)
+            {
+                try { _importConnection?.Close(); _importConnection?.Dispose(); }
+                catch { }
+                _importConnection = null;
+                _importBlogCache = null;
+            }
+        }
+
         public static void EnsureReplyTextColumnExists(string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
@@ -326,15 +356,31 @@ namespace URLNotesGrabberCORE
         public static void AddBlog(string blogName, bool byLikes = false, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
-            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
             if (blogName.Contains("deact"))
                 return;
 
+            SQLiteConnection connection;
+            bool ownsConnection;
+            lock (_importSessionLock)
+            {
+                if (_importConnection != null)
+                {
+                    if (_importBlogCache != null && _importBlogCache.Contains(blogName))
+                        return;
+                    connection = _importConnection;
+                    ownsConnection = false;
+                }
+                else
+                {
+                    connection = new SQLiteConnection("Data Source=" + DBPath);
+                    connection.Open();
+                    ownsConnection = true;
+                }
+            }
+
             try
             {
-                connection.Open();
-
                 // Insert BlogName and DateAdded (current UTC datetime)
                 string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, DateCreated, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @DateCreated, @ByLikes)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -346,6 +392,7 @@ namespace URLNotesGrabberCORE
                     command.Parameters.AddWithValue("@ByLikes", byLikes ? 1 : 0);
                     command.ExecuteNonQuery();
                 }
+                _importBlogCache?.Add(blogName);
                 // Console.WriteLine("+ " + blogName);
             }
             catch (Exception ex)
@@ -356,7 +403,7 @@ namespace URLNotesGrabberCORE
             }
             finally
             {
-                connection.Close();
+                if (ownsConnection) connection.Close();
             }
         }
 
@@ -367,11 +414,17 @@ namespace URLNotesGrabberCORE
             try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
             try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL); } catch { }
 
-            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            SQLiteConnection connection;
+            bool ownsConnection;
+            lock (_importSessionLock)
+            {
+                if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
+                else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
+            }
 
             try
             {
-                connection.Open();
+                if (ownsConnection) connection.Open();
 
                 string sql = @"INSERT INTO Posts (
                                 BlogName,
@@ -418,7 +471,7 @@ namespace URLNotesGrabberCORE
                         ///
                         try
                         {
-                            connection.Open();
+                            if (ownsConnection) connection.Open();
 
                             string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
                             SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
@@ -435,7 +488,7 @@ namespace URLNotesGrabberCORE
                         }
                         finally
                         {
-                            connection.Close();
+                            if (ownsConnection) connection.Close();
                         }
                         ///
                     }
@@ -460,7 +513,7 @@ namespace URLNotesGrabberCORE
             }
             finally
             {
-                connection.Close();
+                if (ownsConnection) connection.Close();
             }
         }
 
@@ -1205,11 +1258,18 @@ namespace URLNotesGrabberCORE
         public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
-            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            SQLiteConnection connection;
+            bool ownsConnection;
+            lock (_importSessionLock)
+            {
+                if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
+                else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
+            }
 
             try
             {
-                connection.Open();
+                if (ownsConnection) connection.Open();
 
                 string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(postDate, '') <> @postDate";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -1229,7 +1289,7 @@ namespace URLNotesGrabberCORE
             }
             finally
             {
-                connection.Close();
+                if (ownsConnection) connection.Close();
             }
         }
 
@@ -1278,11 +1338,18 @@ namespace URLNotesGrabberCORE
         public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
         {
             DBPath ??= GetDefaultDbPath();
-            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            SQLiteConnection connection;
+            bool ownsConnection;
+            lock (_importSessionLock)
+            {
+                if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
+                else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
+            }
 
             try
             {
-                connection.Open();
+                if (ownsConnection) connection.Open();
 
                 string sql = "UPDATE Posts SET ";
                 sql += "postDate = @postDate, ";
@@ -1373,7 +1440,7 @@ namespace URLNotesGrabberCORE
             }
             finally
             {
-                connection.Close();
+                if (ownsConnection) connection.Close();
             }
         }
 

URLNotesGrabberCORE/Program.cs

@@ -124,10 +124,12 @@ namespace URLNotesGrabberCORE
                 try
                 {
                     DataAccess.EnableImportModePragmas();
+                    DataAccess.BeginImportSession();
                     TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports);
                 }
                 finally
                 {
+                    DataAccess.EndImportSession();
                     DataAccess.RestoreImportModePragmas();
                 }
                 Console.WriteLine($"Total posts added: {postsAdded}");