Folds the standalone ThreeTxtFileHelper tool into URLNotesGrabberCORE so text-file ingest/output/correct lives alongside the API scraper. Adds new flags -ingest, -output, -correct (with -apply), -updatepaths, and a one-time -importposts <posts.db> migration. Schema: Blogs.TTFolderPath and Posts.PostType are added by an idempotent migration. On (BlogName, PostID) collisions, content columns are overwritten while engagement columns (ByLikes, RootBlogName, RootURL, HasNotesGathered, NotFound, NotesGatheredDateTime, Likes*) are preserved. Co-Authored-By: Claude Opus 4.7 <[email protected]>
137 lines
6.3 KiB
C#
137 lines
6.3 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
// Port of ThreeTxtFileHelper RunOutputMode + WritePostToFile + RenameExistingTxtFilesToBak.
|
|
// For each Blog with a TTFolderPath, renames any existing .txt files in that folder to .bak,
|
|
// then writes one .txt per PostType containing all posts of that type (date-sorted, fixed
|
|
// field order). Reads from TL.db via DataAccess.GetAllPostsForBlog.
|
|
public static class OutputMode
|
|
{
|
|
public static int Run(IConfiguration config)
|
|
{
|
|
DataAccess.EnsureTTFileHelperColumnsExist();
|
|
|
|
var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
|
|
Console.WriteLine($"Found {blogs.Count} blog(s) to process.");
|
|
|
|
foreach (var (blogName, ttFolderPath) in blogs)
|
|
{
|
|
Console.WriteLine($"\nProcessing blog: {blogName}");
|
|
|
|
if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath))
|
|
{
|
|
Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping.");
|
|
continue;
|
|
}
|
|
|
|
Console.WriteLine($" TTFolderPath: {ttFolderPath}");
|
|
|
|
try
|
|
{
|
|
foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak"))
|
|
File.Delete(bakFile);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($" Error deleting .bak files: {ex.Message}");
|
|
}
|
|
|
|
RenameExistingTxtFilesToBak(ttFolderPath);
|
|
|
|
var posts = DataAccess.GetAllPostsForBlog(blogName);
|
|
Console.WriteLine($" Found {posts.Count} post(s) for this blog.");
|
|
|
|
var grouped = posts.GroupBy(p => p.PostType ?? "Unknown");
|
|
foreach (var typeGroup in grouped)
|
|
{
|
|
string postType = typeGroup.Key ?? "Unknown";
|
|
string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt");
|
|
var ordered = typeGroup.OrderBy(p => p.Date).ToList();
|
|
Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt");
|
|
|
|
using var writer = new StreamWriter(outputFilePath, false, System.Text.Encoding.UTF8);
|
|
bool isFirst = true;
|
|
foreach (var post in ordered)
|
|
{
|
|
if (!isFirst)
|
|
{
|
|
writer.WriteLine();
|
|
writer.WriteLine();
|
|
}
|
|
WritePostToFile(writer, post);
|
|
isFirst = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
Console.WriteLine("\nOutput mode complete.");
|
|
return 0;
|
|
}
|
|
|
|
private static void RenameExistingTxtFilesToBak(string folderPath)
|
|
{
|
|
try
|
|
{
|
|
foreach (var txtFile in Directory.GetFiles(folderPath, "*.txt"))
|
|
{
|
|
string bakPath = Path.ChangeExtension(txtFile, ".bak");
|
|
if (File.Exists(bakPath)) File.Delete(bakPath);
|
|
File.Move(txtFile, bakPath, overwrite: true);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($" Error renaming txt files to .bak: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private static void WritePostToFile(StreamWriter writer, TTPostRecord post)
|
|
{
|
|
var startColumns = new[] { "Post ID", "Date", "Post URL", "Slug", "Reblog Key", "Reblog URL", "Reblog Name", "Title", "Body" };
|
|
var endColumns = new[] { "Tags", "Downloaded Files" };
|
|
|
|
var columns = new Dictionary<string, string>();
|
|
if (!string.IsNullOrWhiteSpace(post.PostId)) columns["Post ID"] = post.PostId;
|
|
if (!string.IsNullOrWhiteSpace(post.Date)) columns["Date"] = post.Date!;
|
|
if (!string.IsNullOrWhiteSpace(post.PostUrl)) columns["Post URL"] = post.PostUrl!;
|
|
if (!string.IsNullOrWhiteSpace(post.Slug)) columns["Slug"] = post.Slug!;
|
|
if (!string.IsNullOrWhiteSpace(post.ReblogKey)) columns["Reblog Key"] = post.ReblogKey!;
|
|
if (!string.IsNullOrWhiteSpace(post.ReblogUrl)) columns["Reblog URL"] = post.ReblogUrl!;
|
|
if (!string.IsNullOrWhiteSpace(post.ReblogName)) columns["Reblog Name"] = post.ReblogName!;
|
|
if (!string.IsNullOrWhiteSpace(post.Title)) columns["Title"] = post.Title!;
|
|
if (!string.IsNullOrWhiteSpace(post.Body)) columns["Body"] = post.Body!;
|
|
if (!string.IsNullOrWhiteSpace(post.HasImage)) columns["Has Image"] = post.HasImage!;
|
|
if (!string.IsNullOrWhiteSpace(post.Summary)) columns["Summary"] = post.Summary!;
|
|
if (!string.IsNullOrWhiteSpace(post.Quote)) columns["Quote"] = post.Quote!;
|
|
if (!string.IsNullOrWhiteSpace(post.Link)) columns["Link"] = post.Link!;
|
|
if (!string.IsNullOrWhiteSpace(post.PhotoUrl)) columns["Photo URL"] = post.PhotoUrl!;
|
|
if (!string.IsNullOrWhiteSpace(post.PhotoCaption)) columns["Photo Caption"] = post.PhotoCaption!;
|
|
if (!string.IsNullOrWhiteSpace(post.AudioCaption)) columns["Audio Caption"] = post.AudioCaption!;
|
|
if (!string.IsNullOrWhiteSpace(post.Question)) columns["Question"] = post.Question!;
|
|
if (!string.IsNullOrWhiteSpace(post.Answer)) columns["Answer"] = post.Answer!;
|
|
if (!string.IsNullOrWhiteSpace(post.Tags)) columns["Tags"] = post.Tags!;
|
|
if (!string.IsNullOrWhiteSpace(post.DownloadedFiles)) columns["Downloaded Files"] = post.DownloadedFiles!;
|
|
|
|
foreach (var col in startColumns)
|
|
{
|
|
if (columns.ContainsKey(col))
|
|
{
|
|
writer.WriteLine($"{col}: {columns[col]}");
|
|
columns.Remove(col);
|
|
}
|
|
}
|
|
|
|
var remaining = columns.Keys.Where(k => !endColumns.Contains(k)).OrderBy(k => k).ToList();
|
|
foreach (var col in remaining)
|
|
writer.WriteLine($"{col}: {columns[col]}");
|
|
|
|
foreach (var col in endColumns)
|
|
{
|
|
if (columns.ContainsKey(col))
|
|
writer.WriteLine($"{col}: {columns[col]}");
|
|
}
|
|
}
|
|
}
|
|
}
|