diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 7b9989b..b2802f0 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -106,6 +106,10 @@ namespace URLNotesGrabberCORE } } + // The database every DataAccess call defaults to, exposed so modes can report + // which file they actually read when their results are surprising. + public static string GetActiveDbPath() => GetDefaultDbPath(); + private static string GetDefaultDbPath() { if (_cachedDbPath != null) @@ -2304,7 +2308,11 @@ namespace URLNotesGrabberCORE }; } - public static void SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null) + // Returns true only when a row's TTFolderPath actually changed. A false means either + // the row already held this value or no row matched the name -- callers must not + // report a write they did not get, which is how a --updatepaths run could once print + // "Updated " for every metadata file while leaving the column entirely NULL. + public static bool SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); try { AddBlog(blogName, false, DBPath); } catch { } @@ -2317,7 +2325,21 @@ namespace URLNotesGrabberCORE cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value); cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); cmd.Parameters.AddWithValue("@name", blogName); - cmd.ExecuteNonQuery(); + return cmd.ExecuteNonQuery() > 0; + } + + // Whether a Blogs row exists under this exact name. BlogName is a BINARY-collated + // primary key, so a metadata filename that differs only in case is a different blog + // as far as the UPDATE above is concerned -- worth telling the user about. + public static bool BlogExists(string blogName, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand("SELECT 1 FROM Blogs WHERE BlogName = @name", connection); + cmd.Parameters.AddWithValue("@name", blogName); + return cmd.ExecuteScalar() != null; } // Partial UPDATE used by the correct-apply path. fieldsToUpdate maps @@ -2391,24 +2413,48 @@ namespace URLNotesGrabberCORE }; } - public static List<(string BlogName, string? TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null) + // Export targets only: active blogs that actually carry a TTFolderPath. + // Blogs is a 144k-row crawl registry and only the few hundred blogs downloaded + // locally have a folder, so returning the unset rows made --output print a skip + // line for every blog Tumblr has ever handed us. + public static List<(string BlogName, string TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - var results = new List<(string, string?)>(); + var results = new List<(string, string)>(); using var connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); - using var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs WHERE IsActive = 1", connection); + using var cmd = new SQLiteCommand( + "SELECT BlogName, TRIM(TTFolderPath) FROM Blogs WHERE IsActive = 1 AND IFNULL(TRIM(TTFolderPath), '') <> '' ORDER BY BlogName", + connection); using var reader = cmd.ExecuteReader(); while (reader.Read()) - { - string name = reader.GetString(0); - string? path = reader.IsDBNull(1) ? null : reader.GetString(1); - results.Add((name, path)); - } + results.Add((reader.GetString(0), reader.GetString(1))); return results; } + // Companion counts for the messages --output and --updatepaths print about coverage. + public static int CountActiveBlogs(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand("SELECT COUNT(*) FROM Blogs WHERE IsActive = 1", connection); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + + public static int CountBlogsWithTTFolderPath(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand( + "SELECT COUNT(*) FROM Blogs WHERE IFNULL(TRIM(TTFolderPath), '') <> ''", connection); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + private static string SafeStr(SQLiteDataReader reader, int ordinal) { return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty; diff --git a/URLNotesGrabberCORE/LegacyPostsDbImporter.cs b/URLNotesGrabberCORE/LegacyPostsDbImporter.cs index a834652..95eef57 100644 --- a/URLNotesGrabberCORE/LegacyPostsDbImporter.cs +++ b/URLNotesGrabberCORE/LegacyPostsDbImporter.cs @@ -29,6 +29,8 @@ namespace URLNotesGrabberCORE Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}"); int blogsCopied = 0; + int blogPathsWritten = 0; + int blogsWithoutPath = 0; int postsUpserted = 0; int errors = 0; @@ -48,7 +50,13 @@ namespace URLNotesGrabberCORE if (string.IsNullOrWhiteSpace(blogName)) continue; try { - DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath); + // A legacy row whose TTFolderPath was already NULL copies nothing. + // Counting it as "copied" is what hid the fact that this import has + // never populated a single path. + if (string.IsNullOrWhiteSpace(ttFolderPath)) + blogsWithoutPath++; + else if (DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath.Trim())) + blogPathsWritten++; blogsCopied++; } catch (Exception ex) @@ -58,7 +66,7 @@ namespace URLNotesGrabberCORE } } } - Console.WriteLine($" Blogs copied: {blogsCopied}"); + Console.WriteLine($" Blogs seen: {blogsCopied}, TTFolderPath written: {blogPathsWritten}, legacy rows with no path: {blogsWithoutPath}"); // 2) Copy Posts try @@ -138,7 +146,8 @@ namespace URLNotesGrabberCORE } Console.WriteLine($"\n========== Legacy import summary =========="); - Console.WriteLine($"Blogs copied: {blogsCopied}"); + Console.WriteLine($"Blogs seen: {blogsCopied}"); + Console.WriteLine($"Paths written: {blogPathsWritten} (legacy rows with no path: {blogsWithoutPath})"); Console.WriteLine($"Posts upserted: {postsUpserted}"); Console.WriteLine($"Errors: {errors}"); return errors == 0 ? 0 : 2; diff --git a/URLNotesGrabberCORE/OutputMode.cs b/URLNotesGrabberCORE/OutputMode.cs index c060880..5096edd 100644 --- a/URLNotesGrabberCORE/OutputMode.cs +++ b/URLNotesGrabberCORE/OutputMode.cs @@ -12,24 +12,44 @@ namespace URLNotesGrabberCORE { DataAccess.EnsureTTFileHelperColumnsExist(); - var blogs = DataAccess.GetAllBlogsWithTTFolderPath(); - Console.WriteLine($"Found {blogs.Count} blog(s) to process."); + string dbPath = DataAccess.GetActiveDbPath(); + Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}"); - foreach (var (blogName, ttFolderPath) in blogs) + var blogs = DataAccess.GetAllBlogsWithTTFolderPath(); + int activeBlogs = DataAccess.CountActiveBlogs(); + Console.WriteLine($"{blogs.Count} of {activeBlogs} active blog(s) have a TTFolderPath."); + + if (blogs.Count == 0) + { + Console.WriteLine($"\nNothing to export: no blog in {Path.GetFullPath(dbPath)} has a TTFolderPath."); + Console.WriteLine("Run --updatepaths on this machine to populate it from \\Index\\*.tumblr / *.tmblrpriv,"); + Console.WriteLine("or set appSettings:PathTTRoot and run --updatepaths with no argument."); + return 1; + } + + int missingFolderCount = 0; + int writtenCount = 0; + + foreach (var (blogName, folder) in blogs) { Console.WriteLine($"\nProcessing blog: {blogName}"); - if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath)) + // A stored path that this machine cannot see means the value was written on + // another machine -- re-running --updatepaths locally is the fix, so say so + // rather than lumping it in with "not set". + if (!Directory.Exists(folder)) { - Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping."); + Console.WriteLine($" TTFolderPath folder not found: {folder}. Skipping."); + missingFolderCount++; continue; } - Console.WriteLine($" TTFolderPath: {ttFolderPath}"); + Console.WriteLine($" TTFolderPath: {folder}"); + writtenCount++; try { - foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak")) + foreach (var bakFile in Directory.GetFiles(folder, "*.bak")) File.Delete(bakFile); } catch (Exception ex) @@ -37,7 +57,7 @@ namespace URLNotesGrabberCORE Console.WriteLine($" Error deleting .bak files: {ex.Message}"); } - RenameExistingTxtFilesToBak(ttFolderPath); + RenameExistingTxtFilesToBak(folder); var posts = DataAccess.GetAllPostsForBlog(blogName); Console.WriteLine($" Found {posts.Count} post(s) for this blog."); @@ -46,7 +66,7 @@ namespace URLNotesGrabberCORE foreach (var typeGroup in grouped) { string postType = typeGroup.Key ?? "Unknown"; - string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt"); + string outputFilePath = Path.Combine(folder, $"{postType}.txt"); var ordered = typeGroup.OrderBy(p => p.Date).ToList(); Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt"); @@ -65,7 +85,11 @@ namespace URLNotesGrabberCORE } } - Console.WriteLine("\nOutput mode complete."); + Console.WriteLine($"\nOutput mode complete. {writtenCount} blog(s) exported, {missingFolderCount} skipped for a missing folder."); + + if (writtenCount == 0) + Console.WriteLine("Every TTFolderPath points at a folder this machine cannot see. The paths were most likely written on another machine -- re-run --updatepaths here so they match local drive letters."); + return 0; } diff --git a/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs b/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs index 47e6548..ce516ea 100644 --- a/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs +++ b/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs @@ -32,6 +32,10 @@ namespace URLNotesGrabberCORE Console.WriteLine($"Found {blogFiles.Count} blog metadata files"); int updatedCount = 0; + int unchangedCount = 0; + int noLocationCount = 0; + int noRowCount = 0; + int errorCount = 0; foreach (var blogFile in blogFiles) { @@ -44,27 +48,56 @@ namespace URLNotesGrabberCORE if (root.TryGetProperty("FileDownloadLocation", out JsonElement locationElement)) { - string? fileDownloadLocation = locationElement.GetString(); + string? fileDownloadLocation = locationElement.GetString()?.Trim(); if (!string.IsNullOrWhiteSpace(fileDownloadLocation)) { - DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation); - updatedCount++; - Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}"); + // Report the database's answer, not the fact that the file parsed. + if (DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation)) + { + updatedCount++; + Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}"); + } + else if (DataAccess.BlogExists(blogName)) + { + unchangedCount++; + } + else + { + noRowCount++; + Console.WriteLine($"No Blogs row named '{blogName}' -- path not stored (name may differ in case)"); + } + } + else + { + noLocationCount++; + Console.WriteLine($"Empty FileDownloadLocation in {blogFile}"); } } else { + noLocationCount++; Console.WriteLine($"No FileDownloadLocation found in {blogFile}"); } } catch (Exception ex) { + errorCount++; Console.WriteLine($"Error processing {blogFile}: {ex.Message}"); } } - Console.WriteLine($"\nUpdated {updatedCount} blogs with TTFolderPath"); - return 0; + Console.WriteLine($"\n========== UpdateBlogPaths summary =========="); + Console.WriteLine($"Metadata files: {blogFiles.Count}"); + Console.WriteLine($"TTFolderPath written: {updatedCount}"); + Console.WriteLine($"Already correct: {unchangedCount}"); + Console.WriteLine($"No FileDownloadLocation: {noLocationCount}"); + Console.WriteLine($"No matching blog row: {noRowCount}"); + Console.WriteLine($"Errors: {errorCount}"); + + int stored = DataAccess.CountBlogsWithTTFolderPath(); + Console.WriteLine($"\nBlogs now holding a TTFolderPath: {stored}"); + + return errorCount == 0 ? 0 : 2; } } }