jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: merge ThreeTxtFileHelper into URLNotesGrabberCORE

Browse Source 
Folds the standalone ThreeTxtFileHelper tool into URLNotesGrabberCORE so
text-file ingest/output/correct lives alongside the API scraper. Adds
new flags -ingest, -output, -correct (with -apply), -updatepaths, and a
one-time -importposts <posts.db> migration.

Schema: Blogs.TTFolderPath and Posts.PostType are added by an idempotent
migration. On (BlogName, PostID) collisions, content columns are
overwritten while engagement columns (ByLikes, RootBlogName, RootURL,
HasNotesGathered, NotFound, NotesGatheredDateTime, Likes*) are preserved.

Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
jim
2026-05-18 12:43:17 -05:00
co-authored by Claude Opus 4.7
parent 5781e121d2
commit 3aff849216
10 changed files with 1419 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/OutputMode.cs
+136
View File
@@ -0,0 +1,136 @@
using Microsoft.Extensions.Configuration;
namespace URLNotesGrabberCORE
{
// Port of ThreeTxtFileHelper RunOutputMode + WritePostToFile + RenameExistingTxtFilesToBak.
// For each Blog with a TTFolderPath, renames any existing .txt files in that folder to .bak,
// then writes one .txt per PostType containing all posts of that type (date-sorted, fixed
// field order). Reads from TL.db via DataAccess.GetAllPostsForBlog.
public static class OutputMode
{
public static int Run(IConfiguration config)
{
DataAccess.EnsureTTFileHelperColumnsExist();
var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
Console.WriteLine($"Found {blogs.Count} blog(s) to process.");
foreach (var (blogName, ttFolderPath) in blogs)
{
Console.WriteLine($"\nProcessing blog: {blogName}");
if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath))
{
Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping.");
continue;
}
Console.WriteLine($" TTFolderPath: {ttFolderPath}");
try
{
foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak"))
File.Delete(bakFile);
}
catch (Exception ex)
{
Console.WriteLine($" Error deleting .bak files: {ex.Message}");
}
RenameExistingTxtFilesToBak(ttFolderPath);
var posts = DataAccess.GetAllPostsForBlog(blogName);
Console.WriteLine($" Found {posts.Count} post(s) for this blog.");
var grouped = posts.GroupBy(p => p.PostType ?? "Unknown");
foreach (var typeGroup in grouped)
{
string postType = typeGroup.Key ?? "Unknown";
string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt");
var ordered = typeGroup.OrderBy(p => p.Date).ToList();
Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt");
using var writer = new StreamWriter(outputFilePath, false, System.Text.Encoding.UTF8);
bool isFirst = true;
foreach (var post in ordered)
{
if (!isFirst)
{
writer.WriteLine();
writer.WriteLine();
}
WritePostToFile(writer, post);
isFirst = false;
}
}
}
Console.WriteLine("\nOutput mode complete.");
return 0;
}
private static void RenameExistingTxtFilesToBak(string folderPath)
{
try
{
foreach (var txtFile in Directory.GetFiles(folderPath, "*.txt"))
{
string bakPath = Path.ChangeExtension(txtFile, ".bak");
if (File.Exists(bakPath)) File.Delete(bakPath);
File.Move(txtFile, bakPath, overwrite: true);
}
}
catch (Exception ex)
{
Console.WriteLine($" Error renaming txt files to .bak: {ex.Message}");
}
}
private static void WritePostToFile(StreamWriter writer, TTPostRecord post)
{
var startColumns = new[] { "Post ID", "Date", "Post URL", "Slug", "Reblog Key", "Reblog URL", "Reblog Name", "Title", "Body" };
var endColumns = new[] { "Tags", "Downloaded Files" };
var columns = new Dictionary<string, string>();
if (!string.IsNullOrWhiteSpace(post.PostId)) columns["Post ID"] = post.PostId;
if (!string.IsNullOrWhiteSpace(post.Date)) columns["Date"] = post.Date!;
if (!string.IsNullOrWhiteSpace(post.PostUrl)) columns["Post URL"] = post.PostUrl!;
if (!string.IsNullOrWhiteSpace(post.Slug)) columns["Slug"] = post.Slug!;
if (!string.IsNullOrWhiteSpace(post.ReblogKey)) columns["Reblog Key"] = post.ReblogKey!;
if (!string.IsNullOrWhiteSpace(post.ReblogUrl)) columns["Reblog URL"] = post.ReblogUrl!;
if (!string.IsNullOrWhiteSpace(post.ReblogName)) columns["Reblog Name"] = post.ReblogName!;
if (!string.IsNullOrWhiteSpace(post.Title)) columns["Title"] = post.Title!;
if (!string.IsNullOrWhiteSpace(post.Body)) columns["Body"] = post.Body!;
if (!string.IsNullOrWhiteSpace(post.HasImage)) columns["Has Image"] = post.HasImage!;
if (!string.IsNullOrWhiteSpace(post.Summary)) columns["Summary"] = post.Summary!;
if (!string.IsNullOrWhiteSpace(post.Quote)) columns["Quote"] = post.Quote!;
if (!string.IsNullOrWhiteSpace(post.Link)) columns["Link"] = post.Link!;
if (!string.IsNullOrWhiteSpace(post.PhotoUrl)) columns["Photo URL"] = post.PhotoUrl!;
if (!string.IsNullOrWhiteSpace(post.PhotoCaption)) columns["Photo Caption"] = post.PhotoCaption!;
if (!string.IsNullOrWhiteSpace(post.AudioCaption)) columns["Audio Caption"] = post.AudioCaption!;
if (!string.IsNullOrWhiteSpace(post.Question)) columns["Question"] = post.Question!;
if (!string.IsNullOrWhiteSpace(post.Answer)) columns["Answer"] = post.Answer!;
if (!string.IsNullOrWhiteSpace(post.Tags)) columns["Tags"] = post.Tags!;
if (!string.IsNullOrWhiteSpace(post.DownloadedFiles)) columns["Downloaded Files"] = post.DownloadedFiles!;
foreach (var col in startColumns)
{
if (columns.ContainsKey(col))
{
writer.WriteLine($"{col}: {columns[col]}");
columns.Remove(col);
}
}
var remaining = columns.Keys.Where(k => !endColumns.Contains(k)).OrderBy(k => k).ToList();
foreach (var col in remaining)
writer.WriteLine($"{col}: {columns[col]}");
foreach (var col in endColumns)
{
if (columns.ContainsKey(col))
writer.WriteLine($"{col}: {columns[col]}");
}
}
}
}