jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'claude/ttfolderpath-not-set-8c9b45' into master

Browse Source
This commit is contained in:
jim
2026-08-05 12:02:58 -05:00
parent ef6629d86a 721224bc13
commit a2763d0026
4 changed files with 141 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+56 -10
View File
@@ -106,6 +106,10 @@ namespace URLNotesGrabberCORE
}
}
// The database every DataAccess call defaults to, exposed so modes can report
// which file they actually read when their results are surprising.
public static string GetActiveDbPath() => GetDefaultDbPath();
private static string GetDefaultDbPath()
{
if (_cachedDbPath != null)
@@ -2304,7 +2308,11 @@ namespace URLNotesGrabberCORE
};
}
public static void SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null)
// Returns true only when a row's TTFolderPath actually changed. A false means either
// the row already held this value or no row matched the name -- callers must not
// report a write they did not get, which is how a --updatepaths run could once print
// "Updated <blog>" for every metadata file while leaving the column entirely NULL.
public static bool SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
try { AddBlog(blogName, false, DBPath); } catch { }
@@ -2317,7 +2325,21 @@ namespace URLNotesGrabberCORE
cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
cmd.Parameters.AddWithValue("@name", blogName);
cmd.ExecuteNonQuery();
return cmd.ExecuteNonQuery() > 0;
}
// Whether a Blogs row exists under this exact name. BlogName is a BINARY-collated
// primary key, so a metadata filename that differs only in case is a different blog
// as far as the UPDATE above is concerned -- worth telling the user about.
public static bool BlogExists(string blogName, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using var cmd = new SQLiteCommand("SELECT 1 FROM Blogs WHERE BlogName = @name", connection);
cmd.Parameters.AddWithValue("@name", blogName);
return cmd.ExecuteScalar() != null;
}
// Partial UPDATE used by the correct-apply path. fieldsToUpdate maps
@@ -2391,24 +2413,48 @@ namespace URLNotesGrabberCORE
};
}
public static List<(string BlogName, string? TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null)
// Export targets only: active blogs that actually carry a TTFolderPath.
// Blogs is a 144k-row crawl registry and only the few hundred blogs downloaded
// locally have a folder, so returning the unset rows made --output print a skip
// line for every blog Tumblr has ever handed us.
public static List<(string BlogName, string TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
var results = new List<(string, string?)>();
var results = new List<(string, string)>();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs WHERE IsActive = 1", connection);
using var cmd = new SQLiteCommand(
"SELECT BlogName, TRIM(TTFolderPath) FROM Blogs WHERE IsActive = 1 AND IFNULL(TRIM(TTFolderPath), '') <> '' ORDER BY BlogName",
connection);
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
string name = reader.GetString(0);
string? path = reader.IsDBNull(1) ? null : reader.GetString(1);
results.Add((name, path));
}
results.Add((reader.GetString(0), reader.GetString(1)));
return results;
}
// Companion counts for the messages --output and --updatepaths print about coverage.
public static int CountActiveBlogs(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using var cmd = new SQLiteCommand("SELECT COUNT(*) FROM Blogs WHERE IsActive = 1", connection);
return Convert.ToInt32(cmd.ExecuteScalar());
}
public static int CountBlogsWithTTFolderPath(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using var cmd = new SQLiteCommand(
"SELECT COUNT(*) FROM Blogs WHERE IFNULL(TRIM(TTFolderPath), '') <> ''", connection);
return Convert.ToInt32(cmd.ExecuteScalar());
}
private static string SafeStr(SQLiteDataReader reader, int ordinal)
{
return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty;
URLNotesGrabberCORE/LegacyPostsDbImporter.cs
+12 -3
View File
@@ -29,6 +29,8 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}");
int blogsCopied = 0;
int blogPathsWritten = 0;
int blogsWithoutPath = 0;
int postsUpserted = 0;
int errors = 0;
@@ -48,7 +50,13 @@ namespace URLNotesGrabberCORE
if (string.IsNullOrWhiteSpace(blogName)) continue;
try
{
DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath);
// A legacy row whose TTFolderPath was already NULL copies nothing.
// Counting it as "copied" is what hid the fact that this import has
// never populated a single path.
if (string.IsNullOrWhiteSpace(ttFolderPath))
blogsWithoutPath++;
else if (DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath.Trim()))
blogPathsWritten++;
blogsCopied++;
}
catch (Exception ex)
@@ -58,7 +66,7 @@ namespace URLNotesGrabberCORE
}
}
}
Console.WriteLine($" Blogs copied: {blogsCopied}");
Console.WriteLine($" Blogs seen: {blogsCopied}, TTFolderPath written: {blogPathsWritten}, legacy rows with no path: {blogsWithoutPath}");
// 2) Copy Posts
try
@@ -138,7 +146,8 @@ namespace URLNotesGrabberCORE
}
Console.WriteLine($"\n========== Legacy import summary ==========");
Console.WriteLine($"Blogs copied: {blogsCopied}");
Console.WriteLine($"Blogs seen: {blogsCopied}");
Console.WriteLine($"Paths written: {blogPathsWritten} (legacy rows with no path: {blogsWithoutPath})");
Console.WriteLine($"Posts upserted: {postsUpserted}");
Console.WriteLine($"Errors: {errors}");
return errors == 0 ? 0 : 2;
URLNotesGrabberCORE/OutputMode.cs
+34 -10
View File
@@ -12,24 +12,44 @@ namespace URLNotesGrabberCORE
{
DataAccess.EnsureTTFileHelperColumnsExist();
var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
Console.WriteLine($"Found {blogs.Count} blog(s) to process.");
string dbPath = DataAccess.GetActiveDbPath();
Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
foreach (var (blogName, ttFolderPath) in blogs)
var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
int activeBlogs = DataAccess.CountActiveBlogs();
Console.WriteLine($"{blogs.Count} of {activeBlogs} active blog(s) have a TTFolderPath.");
if (blogs.Count == 0)
{
Console.WriteLine($"\nNothing to export: no blog in {Path.GetFullPath(dbPath)} has a TTFolderPath.");
Console.WriteLine("Run --updatepaths <root> on this machine to populate it from <root>\\Index\\*.tumblr / *.tmblrpriv,");
Console.WriteLine("or set appSettings:PathTTRoot and run --updatepaths with no argument.");
return 1;
}
int missingFolderCount = 0;
int writtenCount = 0;
foreach (var (blogName, folder) in blogs)
{
Console.WriteLine($"\nProcessing blog: {blogName}");
if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath))
// A stored path that this machine cannot see means the value was written on
// another machine -- re-running --updatepaths locally is the fix, so say so
// rather than lumping it in with "not set".
if (!Directory.Exists(folder))
{
Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping.");
Console.WriteLine($" TTFolderPath folder not found: {folder}. Skipping.");
missingFolderCount++;
continue;
}
Console.WriteLine($" TTFolderPath: {ttFolderPath}");
Console.WriteLine($" TTFolderPath: {folder}");
writtenCount++;
try
{
foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak"))
foreach (var bakFile in Directory.GetFiles(folder, "*.bak"))
File.Delete(bakFile);
}
catch (Exception ex)
@@ -37,7 +57,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine($" Error deleting .bak files: {ex.Message}");
}
RenameExistingTxtFilesToBak(ttFolderPath);
RenameExistingTxtFilesToBak(folder);
var posts = DataAccess.GetAllPostsForBlog(blogName);
Console.WriteLine($" Found {posts.Count} post(s) for this blog.");
@@ -46,7 +66,7 @@ namespace URLNotesGrabberCORE
foreach (var typeGroup in grouped)
{
string postType = typeGroup.Key ?? "Unknown";
string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt");
string outputFilePath = Path.Combine(folder, $"{postType}.txt");
var ordered = typeGroup.OrderBy(p => p.Date).ToList();
Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt");
@@ -65,7 +85,11 @@ namespace URLNotesGrabberCORE
}
}
Console.WriteLine("\nOutput mode complete.");
Console.WriteLine($"\nOutput mode complete. {writtenCount} blog(s) exported, {missingFolderCount} skipped for a missing folder.");
if (writtenCount == 0)
Console.WriteLine("Every TTFolderPath points at a folder this machine cannot see. The paths were most likely written on another machine -- re-run --updatepaths <root> here so they match local drive letters.");
return 0;
}
URLNotesGrabberCORE/UpdateBlogPathsRunner.cs
+39 -6
View File
@@ -32,6 +32,10 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"Found {blogFiles.Count} blog metadata files");
int updatedCount = 0;
int unchangedCount = 0;
int noLocationCount = 0;
int noRowCount = 0;
int errorCount = 0;
foreach (var blogFile in blogFiles)
{
@@ -44,27 +48,56 @@ namespace URLNotesGrabberCORE
if (root.TryGetProperty("FileDownloadLocation", out JsonElement locationElement))
{
string? fileDownloadLocation = locationElement.GetString();
string? fileDownloadLocation = locationElement.GetString()?.Trim();
if (!string.IsNullOrWhiteSpace(fileDownloadLocation))
{
DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation);
updatedCount++;
Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}");
// Report the database's answer, not the fact that the file parsed.
if (DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation))
{
updatedCount++;
Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}");
}
else if (DataAccess.BlogExists(blogName))
{
unchangedCount++;
}
else
{
noRowCount++;
Console.WriteLine($"No Blogs row named '{blogName}' -- path not stored (name may differ in case)");
}
}
else
{
noLocationCount++;
Console.WriteLine($"Empty FileDownloadLocation in {blogFile}");
}
}
else
{
noLocationCount++;
Console.WriteLine($"No FileDownloadLocation found in {blogFile}");
}
}
catch (Exception ex)
{
errorCount++;
Console.WriteLine($"Error processing {blogFile}: {ex.Message}");
}
}
Console.WriteLine($"\nUpdated {updatedCount} blogs with TTFolderPath");
return 0;
Console.WriteLine($"\n========== UpdateBlogPaths summary ==========");
Console.WriteLine($"Metadata files: {blogFiles.Count}");
Console.WriteLine($"TTFolderPath written: {updatedCount}");
Console.WriteLine($"Already correct: {unchangedCount}");
Console.WriteLine($"No FileDownloadLocation: {noLocationCount}");
Console.WriteLine($"No matching blog row: {noRowCount}");
Console.WriteLine($"Errors: {errorCount}");
int stored = DataAccess.CountBlogsWithTTFolderPath();
Console.WriteLine($"\nBlogs now holding a TTFolderPath: {stored}");
return errorCount == 0 ? 0 : 2;
}
}
}