jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: merge ThreeTxtFileHelper into URLNotesGrabberCORE

Browse Source 
Folds the standalone ThreeTxtFileHelper tool into URLNotesGrabberCORE so
text-file ingest/output/correct lives alongside the API scraper. Adds
new flags -ingest, -output, -correct (with -apply), -updatepaths, and a
one-time -importposts <posts.db> migration.

Schema: Blogs.TTFolderPath and Posts.PostType are added by an idempotent
migration. On (BlogName, PostID) collisions, content columns are
overwritten while engagement columns (ByLikes, RootBlogName, RootURL,
HasNotesGathered, NotFound, NotesGatheredDateTime, Likes*) are preserved.

Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
jim
2026-05-18 12:43:17 -05:00
co-authored by Claude Opus 4.7
parent 5781e121d2
commit 3aff849216
10 changed files with 1419 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/LegacyPostsDbImporter.cs
+147
View File
@@ -0,0 +1,147 @@
using System.Data.SQLite;
namespace URLNotesGrabberCORE
{
// One-time migration: opens a legacy ThreeTxtFileHelper posts.db, copies its
// Blog + PostData rows into the merged TL.db via DataAccess.
// Conflict rule on (BlogName, PostId): ThreeTxtFileHelper wins on the 22 content
// columns + PostType + DateModified (handled inside UpsertPostFromTextFile).
// Engagement columns in TL.db (ByLikes, RootBlogName, RootURL, HasNotesGathered,
// NotFound, NotesGatheredDateTime) are preserved.
public static class LegacyPostsDbImporter
{
public static int Run(string legacyDbPath)
{
if (string.IsNullOrWhiteSpace(legacyDbPath))
{
Console.WriteLine("LegacyPostsDbImporter: path to legacy posts.db is required.");
return 1;
}
if (!File.Exists(legacyDbPath))
{
Console.WriteLine($"Legacy posts.db not found at: {legacyDbPath}");
return 1;
}
DataAccess.EnsureTTFileHelperColumnsExist();
Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}");
int blogsCopied = 0;
int postsUpserted = 0;
int errors = 0;
try
{
using var src = new SQLiteConnection("Data Source=" + legacyDbPath + ";Read Only=True;");
src.Open();
// 1) Copy Blogs (BlogName + TTFolderPath)
using (var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs", src))
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
string blogName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0);
string? ttFolderPath = reader.IsDBNull(1) ? null : reader.GetString(1);
if (string.IsNullOrWhiteSpace(blogName)) continue;
try
{
DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath);
blogsCopied++;
}
catch (Exception ex)
{
errors++;
Console.WriteLine($" Blog copy failed for '{blogName}': {ex.Message}");
}
}
}
Console.WriteLine($" Blogs copied: {blogsCopied}");
// 2) Copy Posts
try
{
DataAccess.EnableImportModePragmas();
DataAccess.BeginImportSession();
string sql = @"SELECT BlogName, PostId, ReblogUrl, Date, HasImage, PostUrl, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
PhotoUrl, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType
FROM Posts";
using var cmd = new SQLiteCommand(sql, src);
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
try
{
string blogName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0);
string postId = reader.IsDBNull(1) ? string.Empty : reader.GetString(1);
if (string.IsNullOrWhiteSpace(blogName) || string.IsNullOrWhiteSpace(postId)) continue;
string? hasImageRaw = reader.IsDBNull(4) ? null : reader.GetValue(4)?.ToString();
bool hasImage = !string.IsNullOrWhiteSpace(hasImageRaw)
&& (hasImageRaw.Equals("true", StringComparison.OrdinalIgnoreCase)
|| hasImageRaw == "1"
|| hasImageRaw.Equals("yes", StringComparison.OrdinalIgnoreCase));
DataAccess.UpsertPostFromTextFile(
blogName: blogName,
postID: postId,
reblogURL: reader.IsDBNull(2) ? null : reader.GetString(2),
postDate: reader.IsDBNull(3) ? null : reader.GetString(3),
postURL: reader.IsDBNull(5) ? null : reader.GetString(5),
slug: reader.IsDBNull(6) ? null : reader.GetString(6),
reblogKey: reader.IsDBNull(7) ? null : reader.GetString(7),
reblogName: reader.IsDBNull(8) ? null : reader.GetString(8),
summary: reader.IsDBNull(9) ? null : reader.GetString(9),
quote: reader.IsDBNull(10) ? null : reader.GetString(10),
body: reader.IsDBNull(11) ? null : reader.GetString(11),
tags: reader.IsDBNull(12) ? null : reader.GetString(12),
link: reader.IsDBNull(13) ? null : reader.GetString(13),
photoURL: reader.IsDBNull(14) ? null : reader.GetString(14),
photoCaption: reader.IsDBNull(15) ? null : reader.GetString(15),
downloadedFiles: reader.IsDBNull(16) ? null : reader.GetString(16),
audioCaption: reader.IsDBNull(17) ? null : reader.GetString(17),
question: reader.IsDBNull(18) ? null : reader.GetString(18),
answer: reader.IsDBNull(19) ? null : reader.GetString(19),
title: reader.IsDBNull(20) ? null : reader.GetString(20),
postType: reader.IsDBNull(21) ? null : reader.GetString(21),
hasImage: hasImage);
postsUpserted++;
if (postsUpserted % 500 == 0)
Console.WriteLine($" ... {postsUpserted} posts upserted");
}
catch (Exception ex)
{
errors++;
if (errors < 20)
Console.WriteLine($" Post upsert error: {ex.Message}");
}
}
}
finally
{
DataAccess.EndImportSession();
DataAccess.RestoreImportModePragmas();
}
Console.WriteLine($" Posts upserted: {postsUpserted}");
}
catch (Exception ex)
{
Console.WriteLine($"Fatal error reading legacy posts.db: {ex.Message}");
return 1;
}
Console.WriteLine($"\n========== Legacy import summary ==========");
Console.WriteLine($"Blogs copied: {blogsCopied}");
Console.WriteLine($"Posts upserted: {postsUpserted}");
Console.WriteLine($"Errors: {errors}");
return errors == 0 ? 0 : 2;
}
}
}