Merge branch 'claude/ttfolderpath-not-set-8c9b45' into master

TL.db syncs between machines whose absolute paths differ, so a single
TTFolderPath column cannot be correct on both at once -- the stored paths are
only trustworthy on the machine that wrote them. That made --updatepaths a
mandatory prelude to every --output rather than the one-time setup step it
looks like.

--output now refreshes the column from the TumblThree Index metadata before
exporting. The root comes from the first non-flag argument, else
appSettings:PathTTRoot. With no root available it says so and exports whatever
TL.db already holds; a root whose Index folder is missing is a hard stop, since
silently exporting stale paths is the failure this change exists to prevent.
--norefresh skips the refresh for a pure export.

The scan is extracted from UpdateBlogPathsRunner.Run into a reusable Scan() that
returns counts instead of only printing them, so --updatepaths keeps its
per-file detail while --output prints a single summary line rather than a few
hundred lines ahead of the export.

Verified against a throwaway database: a stale cross-machine path is repaired
and the export lands in the correct local folder; no configured root warns and
continues (exit 0); a missing Index folder stops (exit 1); --norefresh skips the
refresh and exports (exit 0).

Co-Authored-By: Claude Opus 5 <[email protected]>

URLNotesGrabberCORE/DataAccess.cs

@@ -106,6 +106,10 @@ namespace URLNotesGrabberCORE
             }
         }
 
+        // The database every DataAccess call defaults to, exposed so modes can report
+        // which file they actually read when their results are surprising.
+        public static string GetActiveDbPath() => GetDefaultDbPath();
+
         private static string GetDefaultDbPath()
         {
             if (_cachedDbPath != null)
@@ -2304,7 +2308,11 @@ namespace URLNotesGrabberCORE
             };
         }
 
-        public static void SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null)
+        // Returns true only when a row's TTFolderPath actually changed. A false means either
+        // the row already held this value or no row matched the name -- callers must not
+        // report a write they did not get, which is how a --updatepaths run could once print
+        // "Updated <blog>" for every metadata file while leaving the column entirely NULL.
+        public static bool SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
             try { AddBlog(blogName, false, DBPath); } catch { }
@@ -2317,7 +2325,21 @@ namespace URLNotesGrabberCORE
             cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
             cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
             cmd.Parameters.AddWithValue("@name", blogName);
-            cmd.ExecuteNonQuery();
+            return cmd.ExecuteNonQuery() > 0;
+        }
+
+        // Whether a Blogs row exists under this exact name. BlogName is a BINARY-collated
+        // primary key, so a metadata filename that differs only in case is a different blog
+        // as far as the UPDATE above is concerned -- worth telling the user about.
+        public static bool BlogExists(string blogName, string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+
+            using var connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            using var cmd = new SQLiteCommand("SELECT 1 FROM Blogs WHERE BlogName = @name", connection);
+            cmd.Parameters.AddWithValue("@name", blogName);
+            return cmd.ExecuteScalar() != null;
         }
 
         // Partial UPDATE used by the correct-apply path. fieldsToUpdate maps
@@ -2391,24 +2413,48 @@ namespace URLNotesGrabberCORE
             };
         }
 
-        public static List<(string BlogName, string? TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null)
+        // Export targets only: active blogs that actually carry a TTFolderPath.
+        // Blogs is a 144k-row crawl registry and only the few hundred blogs downloaded
+        // locally have a folder, so returning the unset rows made --output print a skip
+        // line for every blog Tumblr has ever handed us.
+        public static List<(string BlogName, string TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
-            var results = new List<(string, string?)>();
+            var results = new List<(string, string)>();
 
             using var connection = new SQLiteConnection("Data Source=" + DBPath);
             connection.Open();
-            using var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs WHERE IsActive = 1", connection);
+            using var cmd = new SQLiteCommand(
+                "SELECT BlogName, TRIM(TTFolderPath) FROM Blogs WHERE IsActive = 1 AND IFNULL(TRIM(TTFolderPath), '') <> '' ORDER BY BlogName",
+                connection);
             using var reader = cmd.ExecuteReader();
             while (reader.Read())
-            {
-                string name = reader.GetString(0);
-                string? path = reader.IsDBNull(1) ? null : reader.GetString(1);
-                results.Add((name, path));
-            }
+                results.Add((reader.GetString(0), reader.GetString(1)));
             return results;
         }
 
+        // Companion counts for the messages --output and --updatepaths print about coverage.
+        public static int CountActiveBlogs(string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+
+            using var connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            using var cmd = new SQLiteCommand("SELECT COUNT(*) FROM Blogs WHERE IsActive = 1", connection);
+            return Convert.ToInt32(cmd.ExecuteScalar());
+        }
+
+        public static int CountBlogsWithTTFolderPath(string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+
+            using var connection = new SQLiteConnection("Data Source=" + DBPath);
+            connection.Open();
+            using var cmd = new SQLiteCommand(
+                "SELECT COUNT(*) FROM Blogs WHERE IFNULL(TRIM(TTFolderPath), '') <> ''", connection);
+            return Convert.ToInt32(cmd.ExecuteScalar());
+        }
+
         private static string SafeStr(SQLiteDataReader reader, int ordinal)
         {
             return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty;

URLNotesGrabberCORE/LegacyPostsDbImporter.cs

@@ -29,6 +29,8 @@ namespace URLNotesGrabberCORE
             Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}");
 
             int blogsCopied = 0;
+            int blogPathsWritten = 0;
+            int blogsWithoutPath = 0;
             int postsUpserted = 0;
             int errors = 0;
 
@@ -48,7 +50,13 @@ namespace URLNotesGrabberCORE
                         if (string.IsNullOrWhiteSpace(blogName)) continue;
                         try
                         {
-                            DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath);
+                            // A legacy row whose TTFolderPath was already NULL copies nothing.
+                            // Counting it as "copied" is what hid the fact that this import has
+                            // never populated a single path.
+                            if (string.IsNullOrWhiteSpace(ttFolderPath))
+                                blogsWithoutPath++;
+                            else if (DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath.Trim()))
+                                blogPathsWritten++;
                             blogsCopied++;
                         }
                         catch (Exception ex)
@@ -58,7 +66,7 @@ namespace URLNotesGrabberCORE
                         }
                     }
                 }
-                Console.WriteLine($"  Blogs copied: {blogsCopied}");
+                Console.WriteLine($"  Blogs seen: {blogsCopied}, TTFolderPath written: {blogPathsWritten}, legacy rows with no path: {blogsWithoutPath}");
 
                 // 2) Copy Posts
                 try
@@ -138,7 +146,8 @@ namespace URLNotesGrabberCORE
             }
 
             Console.WriteLine($"\n========== Legacy import summary ==========");
-            Console.WriteLine($"Blogs copied:    {blogsCopied}");
+            Console.WriteLine($"Blogs seen:      {blogsCopied}");
+            Console.WriteLine($"Paths written:   {blogPathsWritten} (legacy rows with no path: {blogsWithoutPath})");
             Console.WriteLine($"Posts upserted:  {postsUpserted}");
             Console.WriteLine($"Errors:          {errors}");
             return errors == 0 ? 0 : 2;

URLNotesGrabberCORE/OutputMode.cs

@@ -8,28 +8,51 @@ namespace URLNotesGrabberCORE
     // field order). Reads from TL.db via DataAccess.GetAllPostsForBlog.
     public static class OutputMode
     {
-        public static int Run(IConfiguration config)
+        public static int Run(IConfiguration config, string[]? args = null)
         {
             DataAccess.EnsureTTFileHelperColumnsExist();
 
-            var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
-            Console.WriteLine($"Found {blogs.Count} blog(s) to process.");
+            string dbPath = DataAccess.GetActiveDbPath();
+            Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
 
-            foreach (var (blogName, ttFolderPath) in blogs)
+            if (!RefreshPaths(config, args ?? Array.Empty<string>()))
+                return 1;
+
+            var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
+            int activeBlogs = DataAccess.CountActiveBlogs();
+            Console.WriteLine($"{blogs.Count} of {activeBlogs} active blog(s) have a TTFolderPath.");
+
+            if (blogs.Count == 0)
+            {
+                Console.WriteLine($"\nNothing to export: no blog in {Path.GetFullPath(dbPath)} has a TTFolderPath.");
+                Console.WriteLine("Point --output at a TumblThree root so it can populate them: --output <root>,");
+                Console.WriteLine("or set appSettings:PathTTRoot so the refresh runs automatically.");
+                return 1;
+            }
+
+            int missingFolderCount = 0;
+            int writtenCount = 0;
+
+            foreach (var (blogName, folder) in blogs)
             {
                 Console.WriteLine($"\nProcessing blog: {blogName}");
 
-                if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath))
+                // A stored path that this machine cannot see means the value was written on
+                // another machine -- re-running --updatepaths locally is the fix, so say so
+                // rather than lumping it in with "not set".
+                if (!Directory.Exists(folder))
                 {
-                    Console.WriteLine($"  TTFolderPath does not exist or is not set. Skipping.");
+                    Console.WriteLine($"  TTFolderPath folder not found: {folder}. Skipping.");
+                    missingFolderCount++;
                     continue;
                 }
 
-                Console.WriteLine($"  TTFolderPath: {ttFolderPath}");
+                Console.WriteLine($"  TTFolderPath: {folder}");
+                writtenCount++;
 
                 try
                 {
-                    foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak"))
+                    foreach (var bakFile in Directory.GetFiles(folder, "*.bak"))
                         File.Delete(bakFile);
                 }
                 catch (Exception ex)
@@ -37,7 +60,7 @@ namespace URLNotesGrabberCORE
                     Console.WriteLine($"  Error deleting .bak files: {ex.Message}");
                 }
 
-                RenameExistingTxtFilesToBak(ttFolderPath);
+                RenameExistingTxtFilesToBak(folder);
 
                 var posts = DataAccess.GetAllPostsForBlog(blogName);
                 Console.WriteLine($"  Found {posts.Count} post(s) for this blog.");
@@ -46,7 +69,7 @@ namespace URLNotesGrabberCORE
                 foreach (var typeGroup in grouped)
                 {
                     string postType = typeGroup.Key ?? "Unknown";
-                    string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt");
+                    string outputFilePath = Path.Combine(folder, $"{postType}.txt");
                     var ordered = typeGroup.OrderBy(p => p.Date).ToList();
                     Console.WriteLine($"    Writing {ordered.Count} post(s) to {postType}.txt");
 
@@ -65,10 +88,57 @@ namespace URLNotesGrabberCORE
                 }
             }
 
-            Console.WriteLine("\nOutput mode complete.");
+            Console.WriteLine($"\nOutput mode complete. {writtenCount} blog(s) exported, {missingFolderCount} skipped for a missing folder.");
+
+            if (writtenCount == 0)
+                Console.WriteLine("Every TTFolderPath points at a folder this machine cannot see. The paths were most likely written on another machine -- re-run --updatepaths <root> here so they match local drive letters.");
+
             return 0;
         }
 
+        // Re-reads the TumblThree Index metadata into Blogs.TTFolderPath before exporting.
+        // A TL.db synced between machines cannot hold one absolute path that is valid on
+        // both, so the stored paths are only trustworthy on the machine that wrote them --
+        // which makes this refresh part of a normal export rather than a separate chore.
+        // Returns false only when the run should stop.
+        private static bool RefreshPaths(IConfiguration config, string[] args)
+        {
+            var settings = config.GetSection("appSettings");
+
+            if (args.Any(a => string.Equals(a, "--norefresh", StringComparison.OrdinalIgnoreCase)))
+            {
+                Console.WriteLine("Path refresh skipped (--norefresh); exporting to whatever paths TL.db already holds.");
+                return true;
+            }
+
+            string? root = args.FirstOrDefault(a => !a.StartsWith("--", StringComparison.Ordinal))
+                           ?? settings.GetValue<string>("PathTTRoot");
+
+            var result = UpdateBlogPathsRunner.Scan(root, verbose: false);
+
+            switch (result.Outcome)
+            {
+                case UpdateBlogPathsRunner.ScanOutcome.NoRootConfigured:
+                    Console.WriteLine("No TumblThree root configured (appSettings:PathTTRoot is empty and none was passed),");
+                    Console.WriteLine("so TTFolderPath was not refreshed. Pass one as --output <root> to refresh it.");
+                    return true;
+
+                case UpdateBlogPathsRunner.ScanOutcome.IndexFolderMissing:
+                    // Silently exporting stale paths here would defeat the point of folding
+                    // the refresh in, so a bad root is a hard stop.
+                    Console.WriteLine($"Index folder not found at: {result.IndexPath}");
+                    Console.WriteLine("Fix the root (or pass --norefresh to export the paths already in TL.db).");
+                    return false;
+
+                default:
+                    Console.WriteLine($"Refreshed paths from {result.IndexPath}: " +
+                                      $"{result.MetadataFiles} metadata file(s), {result.Written} written, " +
+                                      $"{result.Unchanged} already correct, {result.NoLocation} without a location, " +
+                                      $"{result.NoMatchingRow} without a blog row, {result.Errors} error(s).");
+                    return true;
+            }
+        }
+
         private static void RenameExistingTxtFilesToBak(string folderPath)
         {
             try

URLNotesGrabberCORE/Program.cs

@@ -343,7 +343,7 @@ namespace URLNotesGrabberCORE
                         break;
 
                     case "--output":
-                        exitCode = OutputMode.Run(config);
+                        exitCode = OutputMode.Run(config, args.Skip(1).ToArray());
                         break;
 
                     case "--revert":
@@ -439,7 +439,9 @@ namespace URLNotesGrabberCORE
 
             Console.WriteLine("--ingest [blogname]\t Ingest Tumblr .txt exports from appSettings:PathTTRoot into TL.db (all blogs, or single blog if name given)");
 
-            Console.WriteLine("--output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
+            Console.WriteLine("--output [rootPath]\t Refresh Blogs.TTFolderPath from <root>\\Index (or appSettings:PathTTRoot), then export posts from TL.db back to .txt files in each blog's folder");
+
+            Console.WriteLine("--output --norefresh\t Export without refreshing TTFolderPath first");
 
             Console.WriteLine("--revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
 

URLNotesGrabberCORE/UpdateBlogPathsRunner.cs

@@ -4,34 +4,64 @@ namespace URLNotesGrabberCORE
 {
     // Port of ThreeTxtFileHelper/UpdateBlogPaths.cs. Reads .tumblr / .tmblrpriv metadata
     // files from a root\Index folder and populates Blogs.TTFolderPath in TL.db.
+    //
+    // Scan() is the reusable engine: --updatepaths wraps it as a standalone command and
+    // --output calls it as a refresh step, because a TL.db synced between machines cannot
+    // hold one absolute path that is correct on both.
     public static class UpdateBlogPathsRunner
     {
-        public static int Run(string rootPath)
+        public enum ScanOutcome
+        {
+            Completed,
+            NoRootConfigured,
+            IndexFolderMissing
+        }
+
+        public sealed class ScanResult
+        {
+            public ScanOutcome Outcome { get; init; }
+            public string RootPath { get; init; } = string.Empty;
+            public string IndexPath { get; init; } = string.Empty;
+            public int MetadataFiles { get; init; }
+            public int Written { get; init; }
+            public int Unchanged { get; init; }
+            public int NoLocation { get; init; }
+            public int NoMatchingRow { get; init; }
+            public int Errors { get; init; }
+        }
+
+        // verbose: log a line per metadata file. --updatepaths wants that detail; --output
+        // only wants the counts, since a few hundred lines before the export would bury it.
+        public static ScanResult Scan(string? rootPath, bool verbose)
         {
             if (string.IsNullOrWhiteSpace(rootPath))
-            {
-                Console.WriteLine("UpdateBlogPaths: rootPath is required.");
-                return 1;
-            }
+                return new ScanResult { Outcome = ScanOutcome.NoRootConfigured };
 
             DataAccess.EnsureTTFileHelperColumnsExist();
 
             string indexPath = Path.Combine(rootPath, "Index");
             if (!Directory.Exists(indexPath))
             {
-                Console.WriteLine($"Index folder not found at: {indexPath}");
-                return 1;
+                return new ScanResult
+                {
+                    Outcome = ScanOutcome.IndexFolderMissing,
+                    RootPath = rootPath,
+                    IndexPath = indexPath
+                };
             }
 
-            Console.WriteLine($"Scanning Index folder: {indexPath}");
-
             var blogFiles = Directory.GetFiles(indexPath, "*.tumblr")
                 .Concat(Directory.GetFiles(indexPath, "*.tmblrpriv"))
                 .ToList();
 
+            if (verbose)
                 Console.WriteLine($"Found {blogFiles.Count} blog metadata files");
 
             int updatedCount = 0;
+            int unchangedCount = 0;
+            int noLocationCount = 0;
+            int noRowCount = 0;
+            int errorCount = 0;
 
             foreach (var blogFile in blogFiles)
             {
@@ -44,27 +74,91 @@ namespace URLNotesGrabberCORE
 
                     if (root.TryGetProperty("FileDownloadLocation", out JsonElement locationElement))
                     {
-                        string? fileDownloadLocation = locationElement.GetString();
+                        string? fileDownloadLocation = locationElement.GetString()?.Trim();
                         if (!string.IsNullOrWhiteSpace(fileDownloadLocation))
                         {
-                            DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation);
+                            // Report the database's answer, not the fact that the file parsed.
+                            if (DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation))
+                            {
                                 updatedCount++;
+                                if (verbose)
                                     Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}");
                             }
+                            else if (DataAccess.BlogExists(blogName))
+                            {
+                                unchangedCount++;
+                            }
+                            else
+                            {
+                                noRowCount++;
+                                Console.WriteLine($"No Blogs row named '{blogName}' -- path not stored (name may differ in case)");
+                            }
                         }
                         else
                         {
+                            noLocationCount++;
+                            if (verbose)
+                                Console.WriteLine($"Empty FileDownloadLocation in {blogFile}");
+                        }
+                    }
+                    else
+                    {
+                        noLocationCount++;
+                        if (verbose)
                             Console.WriteLine($"No FileDownloadLocation found in {blogFile}");
                     }
                 }
                 catch (Exception ex)
                 {
+                    errorCount++;
                     Console.WriteLine($"Error processing {blogFile}: {ex.Message}");
                 }
             }
 
-            Console.WriteLine($"\nUpdated {updatedCount} blogs with TTFolderPath");
-            return 0;
+            return new ScanResult
+            {
+                Outcome = ScanOutcome.Completed,
+                RootPath = rootPath,
+                IndexPath = indexPath,
+                MetadataFiles = blogFiles.Count,
+                Written = updatedCount,
+                Unchanged = unchangedCount,
+                NoLocation = noLocationCount,
+                NoMatchingRow = noRowCount,
+                Errors = errorCount
+            };
+        }
+
+        public static int Run(string rootPath)
+        {
+            if (string.IsNullOrWhiteSpace(rootPath))
+            {
+                Console.WriteLine("UpdateBlogPaths: rootPath is required.");
+                return 1;
+            }
+
+            string indexPath = Path.Combine(rootPath, "Index");
+            Console.WriteLine($"Scanning Index folder: {indexPath}");
+
+            var result = Scan(rootPath, verbose: true);
+
+            if (result.Outcome == ScanOutcome.IndexFolderMissing)
+            {
+                Console.WriteLine($"Index folder not found at: {result.IndexPath}");
+                return 1;
+            }
+
+            Console.WriteLine($"\n========== UpdateBlogPaths summary ==========");
+            Console.WriteLine($"Metadata files:           {result.MetadataFiles}");
+            Console.WriteLine($"TTFolderPath written:     {result.Written}");
+            Console.WriteLine($"Already correct:          {result.Unchanged}");
+            Console.WriteLine($"No FileDownloadLocation:  {result.NoLocation}");
+            Console.WriteLine($"No matching blog row:     {result.NoMatchingRow}");
+            Console.WriteLine($"Errors:                   {result.Errors}");
+
+            Console.WriteLine($"\nBlogs now holding a TTFolderPath: {DataAccess.CountBlogsWithTTFolderPath()}");
+
+            return result.Errors == 0 ? 0 : 2;
         }
     }
 }