From 721224bc1377a81e1671545f28414ef00acfc76d Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 5 Aug 2026 12:02:49 -0500 Subject: [PATCH] fix: make --output and --updatepaths tell the truth about TTFolderPath --output iterated all 156k active Blogs rows and printed a "does not exist or is not set" skip line for each, which is nearly every blog in the crawl registry -- only the few hundred downloaded locally ever have a folder. The signal was buried in six figures of noise. GetAllBlogsWithTTFolderPath now selects only active blogs carrying a non-empty path, so --output processes export targets and nothing else. When none exist it says so once, names the database it read, points at --updatepaths, and returns non-zero instead of reporting success. A stored path this machine cannot see is now reported separately from an unset one, with the path shown, because the two are fixed in different places. Paths are trimmed before Directory.Exists, which stray whitespace in a .tumblr FileDownloadLocation would otherwise defeat. Both writers counted optimistically. UpdateBlogPathsRunner printed its per-blog success line and incremented its total from the metadata file parsing, never checking whether the UPDATE matched a row; LegacyPostsDbImporter counted a blog as copied even when the legacy TTFolderPath was NULL. Either could report full success having written nothing -- which is consistent with TL.db holding zero populated paths across all 156,492 active blogs despite 20,679 posts having merged. SetBlogTTFolderPath now returns whether a row changed, and both callers report written / already-correct / no-matching-row separately. Verified against a throwaway database: no-paths case, export case (stale .txt rotated to .bak, per-PostType files, date-sorted), missing-folder case, --updatepaths honest counts, and an idempotent rerun reporting already-correct. Co-Authored-By: Claude Opus 5 --- URLNotesGrabberCORE/DataAccess.cs | 66 +++++++++++++++++--- URLNotesGrabberCORE/LegacyPostsDbImporter.cs | 15 ++++- URLNotesGrabberCORE/OutputMode.cs | 44 ++++++++++--- URLNotesGrabberCORE/UpdateBlogPathsRunner.cs | 45 +++++++++++-- 4 files changed, 141 insertions(+), 29 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 7b9989b..b2802f0 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -106,6 +106,10 @@ namespace URLNotesGrabberCORE } } + // The database every DataAccess call defaults to, exposed so modes can report + // which file they actually read when their results are surprising. + public static string GetActiveDbPath() => GetDefaultDbPath(); + private static string GetDefaultDbPath() { if (_cachedDbPath != null) @@ -2304,7 +2308,11 @@ namespace URLNotesGrabberCORE }; } - public static void SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null) + // Returns true only when a row's TTFolderPath actually changed. A false means either + // the row already held this value or no row matched the name -- callers must not + // report a write they did not get, which is how a --updatepaths run could once print + // "Updated " for every metadata file while leaving the column entirely NULL. + public static bool SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); try { AddBlog(blogName, false, DBPath); } catch { } @@ -2317,7 +2325,21 @@ namespace URLNotesGrabberCORE cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value); cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); cmd.Parameters.AddWithValue("@name", blogName); - cmd.ExecuteNonQuery(); + return cmd.ExecuteNonQuery() > 0; + } + + // Whether a Blogs row exists under this exact name. BlogName is a BINARY-collated + // primary key, so a metadata filename that differs only in case is a different blog + // as far as the UPDATE above is concerned -- worth telling the user about. + public static bool BlogExists(string blogName, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand("SELECT 1 FROM Blogs WHERE BlogName = @name", connection); + cmd.Parameters.AddWithValue("@name", blogName); + return cmd.ExecuteScalar() != null; } // Partial UPDATE used by the correct-apply path. fieldsToUpdate maps @@ -2391,24 +2413,48 @@ namespace URLNotesGrabberCORE }; } - public static List<(string BlogName, string? TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null) + // Export targets only: active blogs that actually carry a TTFolderPath. + // Blogs is a 144k-row crawl registry and only the few hundred blogs downloaded + // locally have a folder, so returning the unset rows made --output print a skip + // line for every blog Tumblr has ever handed us. + public static List<(string BlogName, string TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - var results = new List<(string, string?)>(); + var results = new List<(string, string)>(); using var connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); - using var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs WHERE IsActive = 1", connection); + using var cmd = new SQLiteCommand( + "SELECT BlogName, TRIM(TTFolderPath) FROM Blogs WHERE IsActive = 1 AND IFNULL(TRIM(TTFolderPath), '') <> '' ORDER BY BlogName", + connection); using var reader = cmd.ExecuteReader(); while (reader.Read()) - { - string name = reader.GetString(0); - string? path = reader.IsDBNull(1) ? null : reader.GetString(1); - results.Add((name, path)); - } + results.Add((reader.GetString(0), reader.GetString(1))); return results; } + // Companion counts for the messages --output and --updatepaths print about coverage. + public static int CountActiveBlogs(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand("SELECT COUNT(*) FROM Blogs WHERE IsActive = 1", connection); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + + public static int CountBlogsWithTTFolderPath(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand( + "SELECT COUNT(*) FROM Blogs WHERE IFNULL(TRIM(TTFolderPath), '') <> ''", connection); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + private static string SafeStr(SQLiteDataReader reader, int ordinal) { return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty; diff --git a/URLNotesGrabberCORE/LegacyPostsDbImporter.cs b/URLNotesGrabberCORE/LegacyPostsDbImporter.cs index a834652..95eef57 100644 --- a/URLNotesGrabberCORE/LegacyPostsDbImporter.cs +++ b/URLNotesGrabberCORE/LegacyPostsDbImporter.cs @@ -29,6 +29,8 @@ namespace URLNotesGrabberCORE Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}"); int blogsCopied = 0; + int blogPathsWritten = 0; + int blogsWithoutPath = 0; int postsUpserted = 0; int errors = 0; @@ -48,7 +50,13 @@ namespace URLNotesGrabberCORE if (string.IsNullOrWhiteSpace(blogName)) continue; try { - DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath); + // A legacy row whose TTFolderPath was already NULL copies nothing. + // Counting it as "copied" is what hid the fact that this import has + // never populated a single path. + if (string.IsNullOrWhiteSpace(ttFolderPath)) + blogsWithoutPath++; + else if (DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath.Trim())) + blogPathsWritten++; blogsCopied++; } catch (Exception ex) @@ -58,7 +66,7 @@ namespace URLNotesGrabberCORE } } } - Console.WriteLine($" Blogs copied: {blogsCopied}"); + Console.WriteLine($" Blogs seen: {blogsCopied}, TTFolderPath written: {blogPathsWritten}, legacy rows with no path: {blogsWithoutPath}"); // 2) Copy Posts try @@ -138,7 +146,8 @@ namespace URLNotesGrabberCORE } Console.WriteLine($"\n========== Legacy import summary =========="); - Console.WriteLine($"Blogs copied: {blogsCopied}"); + Console.WriteLine($"Blogs seen: {blogsCopied}"); + Console.WriteLine($"Paths written: {blogPathsWritten} (legacy rows with no path: {blogsWithoutPath})"); Console.WriteLine($"Posts upserted: {postsUpserted}"); Console.WriteLine($"Errors: {errors}"); return errors == 0 ? 0 : 2; diff --git a/URLNotesGrabberCORE/OutputMode.cs b/URLNotesGrabberCORE/OutputMode.cs index c060880..5096edd 100644 --- a/URLNotesGrabberCORE/OutputMode.cs +++ b/URLNotesGrabberCORE/OutputMode.cs @@ -12,24 +12,44 @@ namespace URLNotesGrabberCORE { DataAccess.EnsureTTFileHelperColumnsExist(); - var blogs = DataAccess.GetAllBlogsWithTTFolderPath(); - Console.WriteLine($"Found {blogs.Count} blog(s) to process."); + string dbPath = DataAccess.GetActiveDbPath(); + Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}"); - foreach (var (blogName, ttFolderPath) in blogs) + var blogs = DataAccess.GetAllBlogsWithTTFolderPath(); + int activeBlogs = DataAccess.CountActiveBlogs(); + Console.WriteLine($"{blogs.Count} of {activeBlogs} active blog(s) have a TTFolderPath."); + + if (blogs.Count == 0) + { + Console.WriteLine($"\nNothing to export: no blog in {Path.GetFullPath(dbPath)} has a TTFolderPath."); + Console.WriteLine("Run --updatepaths on this machine to populate it from \\Index\\*.tumblr / *.tmblrpriv,"); + Console.WriteLine("or set appSettings:PathTTRoot and run --updatepaths with no argument."); + return 1; + } + + int missingFolderCount = 0; + int writtenCount = 0; + + foreach (var (blogName, folder) in blogs) { Console.WriteLine($"\nProcessing blog: {blogName}"); - if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath)) + // A stored path that this machine cannot see means the value was written on + // another machine -- re-running --updatepaths locally is the fix, so say so + // rather than lumping it in with "not set". + if (!Directory.Exists(folder)) { - Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping."); + Console.WriteLine($" TTFolderPath folder not found: {folder}. Skipping."); + missingFolderCount++; continue; } - Console.WriteLine($" TTFolderPath: {ttFolderPath}"); + Console.WriteLine($" TTFolderPath: {folder}"); + writtenCount++; try { - foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak")) + foreach (var bakFile in Directory.GetFiles(folder, "*.bak")) File.Delete(bakFile); } catch (Exception ex) @@ -37,7 +57,7 @@ namespace URLNotesGrabberCORE Console.WriteLine($" Error deleting .bak files: {ex.Message}"); } - RenameExistingTxtFilesToBak(ttFolderPath); + RenameExistingTxtFilesToBak(folder); var posts = DataAccess.GetAllPostsForBlog(blogName); Console.WriteLine($" Found {posts.Count} post(s) for this blog."); @@ -46,7 +66,7 @@ namespace URLNotesGrabberCORE foreach (var typeGroup in grouped) { string postType = typeGroup.Key ?? "Unknown"; - string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt"); + string outputFilePath = Path.Combine(folder, $"{postType}.txt"); var ordered = typeGroup.OrderBy(p => p.Date).ToList(); Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt"); @@ -65,7 +85,11 @@ namespace URLNotesGrabberCORE } } - Console.WriteLine("\nOutput mode complete."); + Console.WriteLine($"\nOutput mode complete. {writtenCount} blog(s) exported, {missingFolderCount} skipped for a missing folder."); + + if (writtenCount == 0) + Console.WriteLine("Every TTFolderPath points at a folder this machine cannot see. The paths were most likely written on another machine -- re-run --updatepaths here so they match local drive letters."); + return 0; } diff --git a/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs b/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs index 47e6548..ce516ea 100644 --- a/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs +++ b/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs @@ -32,6 +32,10 @@ namespace URLNotesGrabberCORE Console.WriteLine($"Found {blogFiles.Count} blog metadata files"); int updatedCount = 0; + int unchangedCount = 0; + int noLocationCount = 0; + int noRowCount = 0; + int errorCount = 0; foreach (var blogFile in blogFiles) { @@ -44,27 +48,56 @@ namespace URLNotesGrabberCORE if (root.TryGetProperty("FileDownloadLocation", out JsonElement locationElement)) { - string? fileDownloadLocation = locationElement.GetString(); + string? fileDownloadLocation = locationElement.GetString()?.Trim(); if (!string.IsNullOrWhiteSpace(fileDownloadLocation)) { - DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation); - updatedCount++; - Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}"); + // Report the database's answer, not the fact that the file parsed. + if (DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation)) + { + updatedCount++; + Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}"); + } + else if (DataAccess.BlogExists(blogName)) + { + unchangedCount++; + } + else + { + noRowCount++; + Console.WriteLine($"No Blogs row named '{blogName}' -- path not stored (name may differ in case)"); + } + } + else + { + noLocationCount++; + Console.WriteLine($"Empty FileDownloadLocation in {blogFile}"); } } else { + noLocationCount++; Console.WriteLine($"No FileDownloadLocation found in {blogFile}"); } } catch (Exception ex) { + errorCount++; Console.WriteLine($"Error processing {blogFile}: {ex.Message}"); } } - Console.WriteLine($"\nUpdated {updatedCount} blogs with TTFolderPath"); - return 0; + Console.WriteLine($"\n========== UpdateBlogPaths summary =========="); + Console.WriteLine($"Metadata files: {blogFiles.Count}"); + Console.WriteLine($"TTFolderPath written: {updatedCount}"); + Console.WriteLine($"Already correct: {unchangedCount}"); + Console.WriteLine($"No FileDownloadLocation: {noLocationCount}"); + Console.WriteLine($"No matching blog row: {noRowCount}"); + Console.WriteLine($"Errors: {errorCount}"); + + int stored = DataAccess.CountBlogsWithTTFolderPath(); + Console.WriteLine($"\nBlogs now holding a TTFolderPath: {stored}"); + + return errorCount == 0 ? 0 : 2; } } }