feat: merge ThreeTxtFileHelper into URLNotesGrabberCORE
Folds the standalone ThreeTxtFileHelper tool into URLNotesGrabberCORE so text-file ingest/output/correct lives alongside the API scraper. Adds new flags -ingest, -output, -correct (with -apply), -updatepaths, and a one-time -importposts <posts.db> migration. Schema: Blogs.TTFolderPath and Posts.PostType are added by an idempotent migration. On (BlogName, PostID) collisions, content columns are overwritten while engagement columns (ByLikes, RootBlogName, RootURL, HasNotesGathered, NotFound, NotesGatheredDateTime, Likes*) are preserved. Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
Claude Opus 4.7
@@ -0,0 +1,277 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace URLNotesGrabberCORE
|
||||
{
|
||||
// Port of ThreeTxtFileHelper RunCorrectionMode (dry-run) and RunFullCorrectionMode (apply).
|
||||
// Scans a BAK directory of .txt files, parses posts with multi-line field support,
|
||||
// and either reports or applies content-column corrections to TL.db.Posts.
|
||||
// Apply path only writes non-empty values (mirrors original ThreeTxtFileHelper semantics)
|
||||
// and never touches engagement columns.
|
||||
public static class CorrectMode
|
||||
{
|
||||
public static int Run(IConfiguration config, string[] args, bool applyChanges)
|
||||
{
|
||||
DataAccess.EnsureTTFileHelperColumnsExist();
|
||||
|
||||
// Resolve BAK path: explicit arg > appSettings:PathTTBackup > derive from PathTTRoot/PathInput
|
||||
string? bakRootPath = args.Length > 0 ? args[0] : config["appSettings:PathTTBackup"];
|
||||
if (string.IsNullOrWhiteSpace(bakRootPath))
|
||||
{
|
||||
string? root = config["appSettings:PathTTRoot"];
|
||||
if (string.IsNullOrWhiteSpace(root)) root = config["appSettings:PathInput"];
|
||||
if (!string.IsNullOrWhiteSpace(root))
|
||||
bakRootPath = root.TrimEnd('\\', '/') + "_BAK\\";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(bakRootPath) || !Directory.Exists(bakRootPath))
|
||||
{
|
||||
Console.WriteLine($"BAK directory not found: {bakRootPath}");
|
||||
return 1;
|
||||
}
|
||||
|
||||
Console.WriteLine($"BAK source path: {bakRootPath}");
|
||||
Console.WriteLine(applyChanges
|
||||
? "Correction mode: APPLY - non-empty fields from BAK overwrite DB columns\n"
|
||||
: "Correction mode: Dry-run - reports multi-line field updates available\n");
|
||||
|
||||
string prefixesPath = config["appSettings:PathPrefixes"] ?? "prefixes.txt";
|
||||
if (!Path.IsPathRooted(prefixesPath))
|
||||
prefixesPath = Path.Combine(AppContext.BaseDirectory, prefixesPath);
|
||||
if (!File.Exists(prefixesPath))
|
||||
{
|
||||
Console.WriteLine($"Prefixes file not found: {prefixesPath}");
|
||||
return 1;
|
||||
}
|
||||
var allowedPrefixes = new HashSet<string>(File.ReadLines(prefixesPath), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (applyChanges)
|
||||
{
|
||||
Console.Write("WARNING: This will overwrite non-empty fields in matching posts from BAK files. Continue? (yes/no): ");
|
||||
string? response = Console.ReadLine();
|
||||
if (string.IsNullOrWhiteSpace(response) || !response.Equals("yes", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Console.WriteLine("Operation cancelled.");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
var bakTxtFiles = new List<string>();
|
||||
try
|
||||
{
|
||||
foreach (var dir in Directory.GetDirectories(bakRootPath, "*", SearchOption.AllDirectories))
|
||||
bakTxtFiles.AddRange(Directory.GetFiles(dir, "*.txt"));
|
||||
bakTxtFiles.AddRange(Directory.GetFiles(bakRootPath, "*.txt"));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error scanning BAK directory: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
Console.WriteLine($"Found {bakTxtFiles.Count} file(s) in BAK directory\n");
|
||||
|
||||
int totalPostsFound = 0;
|
||||
int postsWithUpdates = 0;
|
||||
int postsUpdated = 0;
|
||||
int postsNotFound = 0;
|
||||
var correctionLog = new List<string>();
|
||||
var updateLog = new List<string>();
|
||||
|
||||
foreach (string bakFile in bakTxtFiles)
|
||||
{
|
||||
Console.WriteLine($"Processing BAK file: {Path.GetFileName(bakFile)}");
|
||||
try
|
||||
{
|
||||
var bakPosts = ParsePostsFromFile(bakFile, allowedPrefixes);
|
||||
Console.WriteLine($" Found {bakPosts.Count} post(s) in this file");
|
||||
|
||||
foreach (var (postId, bakData) in bakPosts)
|
||||
{
|
||||
totalPostsFound++;
|
||||
var dbPost = DataAccess.GetPostByIdAnyBlog(postId);
|
||||
if (dbPost == null)
|
||||
{
|
||||
postsNotFound++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (applyChanges)
|
||||
{
|
||||
bool updated = DataAccess.UpdatePostContentFields(dbPost.BlogName, dbPost.PostId, bakData);
|
||||
if (updated)
|
||||
{
|
||||
postsUpdated++;
|
||||
updateLog.Add($"Post ID: {postId} - Updated from {Path.GetFileName(bakFile)}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var updateList = BuildDryRunDiff(bakData, dbPost);
|
||||
if (updateList.Count > 0)
|
||||
{
|
||||
postsWithUpdates++;
|
||||
correctionLog.Add($"\nPost ID: {postId}");
|
||||
correctionLog.Add($" File: {Path.GetFileName(bakFile)}");
|
||||
correctionLog.Add($" Fields to update:");
|
||||
correctionLog.AddRange(updateList);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($" ERROR processing file: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
if (applyChanges)
|
||||
{
|
||||
Console.WriteLine($"\n========== CORRECTION COMPLETE ==========");
|
||||
Console.WriteLine($"Total posts found in BAK files: {totalPostsFound}");
|
||||
Console.WriteLine($"Posts updated in database: {postsUpdated}");
|
||||
Console.WriteLine($"Posts not found in database: {postsNotFound}");
|
||||
|
||||
string logPath = config["appSettings:PathCorrectionApplied"] ?? "correction_applied.txt";
|
||||
try
|
||||
{
|
||||
var logLines = new List<string>
|
||||
{
|
||||
$"Correction Applied: {DateTime.Now:yyyy-MM-dd HH:mm:ss}",
|
||||
$"Total posts found in BAK files: {totalPostsFound}",
|
||||
$"Posts updated in database: {postsUpdated}",
|
||||
$"Posts not found in database: {postsNotFound}",
|
||||
"",
|
||||
"Updated Posts:"
|
||||
};
|
||||
logLines.AddRange(updateLog);
|
||||
File.WriteAllLines(logPath, logLines);
|
||||
Console.WriteLine($"Update log saved to: {logPath}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error writing log file: {ex.Message}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"\n========== CORRECTION REPORT (DRY RUN) ==========");
|
||||
Console.WriteLine($"Total posts found in BAK files: {totalPostsFound}");
|
||||
Console.WriteLine($"Posts with multi-line field updates available: {postsWithUpdates}");
|
||||
|
||||
if (correctionLog.Count > 0)
|
||||
{
|
||||
string logPath = config["appSettings:PathCorrectionReport"] ?? "correction_report.txt";
|
||||
try
|
||||
{
|
||||
File.WriteAllLines(logPath, correctionLog);
|
||||
Console.WriteLine($"\nDetailed report saved to: {logPath}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error writing report file: {ex.Message}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("\nNo multi-line field updates found.");
|
||||
}
|
||||
|
||||
Console.WriteLine("\nDry-run complete. No database changes were made.");
|
||||
Console.WriteLine("If updates look correct, re-run with `-correct -apply` to apply changes.");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static List<string> BuildDryRunDiff(Dictionary<string, string> bakData, TTPostRecord dbPost)
|
||||
{
|
||||
var updates = new List<string>();
|
||||
foreach (var (fieldName, bakValue) in bakData)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bakValue)) continue;
|
||||
string? currentValue = fieldName.ToLowerInvariant() switch
|
||||
{
|
||||
"reblog url" => dbPost.ReblogUrl,
|
||||
"date" => dbPost.Date,
|
||||
"has image" => dbPost.HasImage,
|
||||
"post url" => dbPost.PostUrl,
|
||||
"slug" => dbPost.Slug,
|
||||
"reblog key" => dbPost.ReblogKey,
|
||||
"reblog name" => dbPost.ReblogName,
|
||||
"summary" => dbPost.Summary,
|
||||
"quote" => dbPost.Quote,
|
||||
"body" => dbPost.Body,
|
||||
"tags" => dbPost.Tags,
|
||||
"link" => dbPost.Link,
|
||||
"photo url" => dbPost.PhotoUrl,
|
||||
"photo caption" => dbPost.PhotoCaption,
|
||||
"downloaded files" => dbPost.DownloadedFiles,
|
||||
"audio caption" => dbPost.AudioCaption,
|
||||
"question" => dbPost.Question,
|
||||
"answer" => dbPost.Answer,
|
||||
"title" => dbPost.Title,
|
||||
_ => null
|
||||
};
|
||||
if (bakValue != currentValue && bakValue.Contains('\n'))
|
||||
updates.Add($" {fieldName}: [MULTILINE]");
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
|
||||
private static Dictionary<string, Dictionary<string, string>> ParsePostsFromFile(string filePath, HashSet<string> allowedPrefixes)
|
||||
{
|
||||
var posts = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
|
||||
var lines = File.ReadAllLines(filePath);
|
||||
int lineIndex = 0;
|
||||
string currentPostId = "";
|
||||
var currentPostData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
while (lineIndex < lines.Length)
|
||||
{
|
||||
string line = lines[lineIndex];
|
||||
string searchText = line.Length > 25 ? line.Substring(0, 25) : line;
|
||||
int colonIndex = searchText.IndexOf(": ");
|
||||
|
||||
if (colonIndex > 0)
|
||||
{
|
||||
string prefix = line.Substring(0, colonIndex).Trim();
|
||||
if (!string.IsNullOrWhiteSpace(prefix) && allowedPrefixes.Contains(prefix))
|
||||
{
|
||||
if (string.Equals(prefix, "Post ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0)
|
||||
posts[currentPostId] = new Dictionary<string, string>(currentPostData, StringComparer.OrdinalIgnoreCase);
|
||||
currentPostId = line.Substring(colonIndex + 2).Trim();
|
||||
currentPostData.Clear();
|
||||
lineIndex++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var valueLines = new List<string> { line.Substring(colonIndex + 2).Trim() };
|
||||
int nextLineIndex = lineIndex + 1;
|
||||
while (nextLineIndex < lines.Length)
|
||||
{
|
||||
string nextLine = lines[nextLineIndex];
|
||||
string nextSearch = nextLine.Length > 25 ? nextLine.Substring(0, 25) : nextLine;
|
||||
int nextColon = nextSearch.IndexOf(": ");
|
||||
if (nextColon > 0)
|
||||
{
|
||||
string nextPrefix = nextLine.Substring(0, nextColon).Trim();
|
||||
if (!string.IsNullOrWhiteSpace(nextPrefix) && allowedPrefixes.Contains(nextPrefix))
|
||||
break;
|
||||
}
|
||||
valueLines.Add(nextLine);
|
||||
nextLineIndex++;
|
||||
}
|
||||
currentPostData[prefix] = string.Join("\n", valueLines);
|
||||
lineIndex = nextLineIndex;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
lineIndex++;
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0)
|
||||
posts[currentPostId] = new Dictionary<string, string>(currentPostData, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
return posts;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user