--output iterated all 156k active Blogs rows and printed a "does not exist or is not set" skip line for each, which is nearly every blog in the crawl registry -- only the few hundred downloaded locally ever have a folder. The signal was buried in six figures of noise. GetAllBlogsWithTTFolderPath now selects only active blogs carrying a non-empty path, so --output processes export targets and nothing else. When none exist it says so once, names the database it read, points at --updatepaths, and returns non-zero instead of reporting success. A stored path this machine cannot see is now reported separately from an unset one, with the path shown, because the two are fixed in different places. Paths are trimmed before Directory.Exists, which stray whitespace in a .tumblr FileDownloadLocation would otherwise defeat. Both writers counted optimistically. UpdateBlogPathsRunner printed its per-blog success line and incremented its total from the metadata file parsing, never checking whether the UPDATE matched a row; LegacyPostsDbImporter counted a blog as copied even when the legacy TTFolderPath was NULL. Either could report full success having written nothing -- which is consistent with TL.db holding zero populated paths across all 156,492 active blogs despite 20,679 posts having merged. SetBlogTTFolderPath now returns whether a row changed, and both callers report written / already-correct / no-matching-row separately. Verified against a throwaway database: no-paths case, export case (stale .txt rotated to .bak, per-PostType files, date-sorted), missing-folder case, --updatepaths honest counts, and an idempotent rerun reporting already-correct. Co-Authored-By: Claude Opus 5 <[email protected]>
161 lines
7.6 KiB
C#
161 lines
7.6 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
// Port of ThreeTxtFileHelper RunOutputMode + WritePostToFile + RenameExistingTxtFilesToBak.
|
|
// For each Blog with a TTFolderPath, renames any existing .txt files in that folder to .bak,
|
|
// then writes one .txt per PostType containing all posts of that type (date-sorted, fixed
|
|
// field order). Reads from TL.db via DataAccess.GetAllPostsForBlog.
|
|
public static class OutputMode
|
|
{
|
|
public static int Run(IConfiguration config)
|
|
{
|
|
DataAccess.EnsureTTFileHelperColumnsExist();
|
|
|
|
string dbPath = DataAccess.GetActiveDbPath();
|
|
Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
|
|
|
|
var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
|
|
int activeBlogs = DataAccess.CountActiveBlogs();
|
|
Console.WriteLine($"{blogs.Count} of {activeBlogs} active blog(s) have a TTFolderPath.");
|
|
|
|
if (blogs.Count == 0)
|
|
{
|
|
Console.WriteLine($"\nNothing to export: no blog in {Path.GetFullPath(dbPath)} has a TTFolderPath.");
|
|
Console.WriteLine("Run --updatepaths <root> on this machine to populate it from <root>\\Index\\*.tumblr / *.tmblrpriv,");
|
|
Console.WriteLine("or set appSettings:PathTTRoot and run --updatepaths with no argument.");
|
|
return 1;
|
|
}
|
|
|
|
int missingFolderCount = 0;
|
|
int writtenCount = 0;
|
|
|
|
foreach (var (blogName, folder) in blogs)
|
|
{
|
|
Console.WriteLine($"\nProcessing blog: {blogName}");
|
|
|
|
// A stored path that this machine cannot see means the value was written on
|
|
// another machine -- re-running --updatepaths locally is the fix, so say so
|
|
// rather than lumping it in with "not set".
|
|
if (!Directory.Exists(folder))
|
|
{
|
|
Console.WriteLine($" TTFolderPath folder not found: {folder}. Skipping.");
|
|
missingFolderCount++;
|
|
continue;
|
|
}
|
|
|
|
Console.WriteLine($" TTFolderPath: {folder}");
|
|
writtenCount++;
|
|
|
|
try
|
|
{
|
|
foreach (var bakFile in Directory.GetFiles(folder, "*.bak"))
|
|
File.Delete(bakFile);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($" Error deleting .bak files: {ex.Message}");
|
|
}
|
|
|
|
RenameExistingTxtFilesToBak(folder);
|
|
|
|
var posts = DataAccess.GetAllPostsForBlog(blogName);
|
|
Console.WriteLine($" Found {posts.Count} post(s) for this blog.");
|
|
|
|
var grouped = posts.GroupBy(p => p.PostType ?? "Unknown");
|
|
foreach (var typeGroup in grouped)
|
|
{
|
|
string postType = typeGroup.Key ?? "Unknown";
|
|
string outputFilePath = Path.Combine(folder, $"{postType}.txt");
|
|
var ordered = typeGroup.OrderBy(p => p.Date).ToList();
|
|
Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt");
|
|
|
|
using var writer = new StreamWriter(outputFilePath, false, System.Text.Encoding.UTF8);
|
|
bool isFirst = true;
|
|
foreach (var post in ordered)
|
|
{
|
|
if (!isFirst)
|
|
{
|
|
writer.WriteLine();
|
|
writer.WriteLine();
|
|
}
|
|
WritePostToFile(writer, post);
|
|
isFirst = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"\nOutput mode complete. {writtenCount} blog(s) exported, {missingFolderCount} skipped for a missing folder.");
|
|
|
|
if (writtenCount == 0)
|
|
Console.WriteLine("Every TTFolderPath points at a folder this machine cannot see. The paths were most likely written on another machine -- re-run --updatepaths <root> here so they match local drive letters.");
|
|
|
|
return 0;
|
|
}
|
|
|
|
private static void RenameExistingTxtFilesToBak(string folderPath)
|
|
{
|
|
try
|
|
{
|
|
foreach (var txtFile in Directory.GetFiles(folderPath, "*.txt"))
|
|
{
|
|
string bakPath = Path.ChangeExtension(txtFile, ".bak");
|
|
if (File.Exists(bakPath)) File.Delete(bakPath);
|
|
File.Move(txtFile, bakPath, overwrite: true);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($" Error renaming txt files to .bak: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private static void WritePostToFile(StreamWriter writer, TTPostRecord post)
|
|
{
|
|
var startColumns = new[] { "Post ID", "Date", "Post URL", "Slug", "Reblog Key", "Reblog URL", "Reblog Name", "Title", "Body" };
|
|
var endColumns = new[] { "Tags", "Downloaded Files" };
|
|
|
|
var columns = new Dictionary<string, string>();
|
|
if (!string.IsNullOrWhiteSpace(post.PostId)) columns["Post ID"] = post.PostId;
|
|
if (!string.IsNullOrWhiteSpace(post.Date)) columns["Date"] = post.Date!;
|
|
if (!string.IsNullOrWhiteSpace(post.PostUrl)) columns["Post URL"] = post.PostUrl!;
|
|
if (!string.IsNullOrWhiteSpace(post.Slug)) columns["Slug"] = post.Slug!;
|
|
if (!string.IsNullOrWhiteSpace(post.ReblogKey)) columns["Reblog Key"] = post.ReblogKey!;
|
|
if (!string.IsNullOrWhiteSpace(post.ReblogUrl)) columns["Reblog URL"] = post.ReblogUrl!;
|
|
if (!string.IsNullOrWhiteSpace(post.ReblogName)) columns["Reblog Name"] = post.ReblogName!;
|
|
if (!string.IsNullOrWhiteSpace(post.Title)) columns["Title"] = post.Title!;
|
|
if (!string.IsNullOrWhiteSpace(post.Body)) columns["Body"] = post.Body!;
|
|
if (!string.IsNullOrWhiteSpace(post.HasImage)) columns["Has Image"] = post.HasImage!;
|
|
if (!string.IsNullOrWhiteSpace(post.Summary)) columns["Summary"] = post.Summary!;
|
|
if (!string.IsNullOrWhiteSpace(post.Quote)) columns["Quote"] = post.Quote!;
|
|
if (!string.IsNullOrWhiteSpace(post.Link)) columns["Link"] = post.Link!;
|
|
if (!string.IsNullOrWhiteSpace(post.PhotoUrl)) columns["Photo URL"] = post.PhotoUrl!;
|
|
if (!string.IsNullOrWhiteSpace(post.PhotoCaption)) columns["Photo Caption"] = post.PhotoCaption!;
|
|
if (!string.IsNullOrWhiteSpace(post.AudioCaption)) columns["Audio Caption"] = post.AudioCaption!;
|
|
if (!string.IsNullOrWhiteSpace(post.Question)) columns["Question"] = post.Question!;
|
|
if (!string.IsNullOrWhiteSpace(post.Answer)) columns["Answer"] = post.Answer!;
|
|
if (!string.IsNullOrWhiteSpace(post.Tags)) columns["Tags"] = post.Tags!;
|
|
if (!string.IsNullOrWhiteSpace(post.DownloadedFiles)) columns["Downloaded Files"] = post.DownloadedFiles!;
|
|
|
|
foreach (var col in startColumns)
|
|
{
|
|
if (columns.ContainsKey(col))
|
|
{
|
|
writer.WriteLine($"{col}: {columns[col]}");
|
|
columns.Remove(col);
|
|
}
|
|
}
|
|
|
|
var remaining = columns.Keys.Where(k => !endColumns.Contains(k)).OrderBy(k => k).ToList();
|
|
foreach (var col in remaining)
|
|
writer.WriteLine($"{col}: {columns[col]}");
|
|
|
|
foreach (var col in endColumns)
|
|
{
|
|
if (columns.ContainsKey(col))
|
|
writer.WriteLine($"{col}: {columns[col]}");
|
|
}
|
|
}
|
|
}
|
|
}
|