jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: merge ThreeTxtFileHelper into URLNotesGrabberCORE

Browse Source 
Folds the standalone ThreeTxtFileHelper tool into URLNotesGrabberCORE so
text-file ingest/output/correct lives alongside the API scraper. Adds
new flags -ingest, -output, -correct (with -apply), -updatepaths, and a
one-time -importposts <posts.db> migration.

Schema: Blogs.TTFolderPath and Posts.PostType are added by an idempotent
migration. On (BlogName, PostID) collisions, content columns are
overwritten while engagement columns (ByLikes, RootBlogName, RootURL,
HasNotesGathered, NotFound, NotesGatheredDateTime, Likes*) are preserved.

Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
jim
2026-05-18 12:43:17 -05:00
co-authored by Claude Opus 4.7
parent 5781e121d2
commit 3aff849216
10 changed files with 1419 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/CorrectMode.cs
+277
View File
@@ -0,0 +1,277 @@
using Microsoft.Extensions.Configuration;
namespace URLNotesGrabberCORE
{
// Port of ThreeTxtFileHelper RunCorrectionMode (dry-run) and RunFullCorrectionMode (apply).
// Scans a BAK directory of .txt files, parses posts with multi-line field support,
// and either reports or applies content-column corrections to TL.db.Posts.
// Apply path only writes non-empty values (mirrors original ThreeTxtFileHelper semantics)
// and never touches engagement columns.
public static class CorrectMode
{
public static int Run(IConfiguration config, string[] args, bool applyChanges)
{
DataAccess.EnsureTTFileHelperColumnsExist();
// Resolve BAK path: explicit arg > appSettings:PathTTBackup > derive from PathTTRoot/PathInput
string? bakRootPath = args.Length > 0 ? args[0] : config["appSettings:PathTTBackup"];
if (string.IsNullOrWhiteSpace(bakRootPath))
{
string? root = config["appSettings:PathTTRoot"];
if (string.IsNullOrWhiteSpace(root)) root = config["appSettings:PathInput"];
if (!string.IsNullOrWhiteSpace(root))
bakRootPath = root.TrimEnd('\\', '/') + "_BAK\\";
}
if (string.IsNullOrWhiteSpace(bakRootPath) || !Directory.Exists(bakRootPath))
{
Console.WriteLine($"BAK directory not found: {bakRootPath}");
return 1;
}
Console.WriteLine($"BAK source path: {bakRootPath}");
Console.WriteLine(applyChanges
? "Correction mode: APPLY - non-empty fields from BAK overwrite DB columns\n"
: "Correction mode: Dry-run - reports multi-line field updates available\n");
string prefixesPath = config["appSettings:PathPrefixes"] ?? "prefixes.txt";
if (!Path.IsPathRooted(prefixesPath))
prefixesPath = Path.Combine(AppContext.BaseDirectory, prefixesPath);
if (!File.Exists(prefixesPath))
{
Console.WriteLine($"Prefixes file not found: {prefixesPath}");
return 1;
}
var allowedPrefixes = new HashSet<string>(File.ReadLines(prefixesPath), StringComparer.OrdinalIgnoreCase);
if (applyChanges)
{
Console.Write("WARNING: This will overwrite non-empty fields in matching posts from BAK files. Continue? (yes/no): ");
string? response = Console.ReadLine();
if (string.IsNullOrWhiteSpace(response) || !response.Equals("yes", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine("Operation cancelled.");
return 0;
}
}
var bakTxtFiles = new List<string>();
try
{
foreach (var dir in Directory.GetDirectories(bakRootPath, "*", SearchOption.AllDirectories))
bakTxtFiles.AddRange(Directory.GetFiles(dir, "*.txt"));
bakTxtFiles.AddRange(Directory.GetFiles(bakRootPath, "*.txt"));
}
catch (Exception ex)
{
Console.WriteLine($"Error scanning BAK directory: {ex.Message}");
return 1;
}
Console.WriteLine($"Found {bakTxtFiles.Count} file(s) in BAK directory\n");
int totalPostsFound = 0;
int postsWithUpdates = 0;
int postsUpdated = 0;
int postsNotFound = 0;
var correctionLog = new List<string>();
var updateLog = new List<string>();
foreach (string bakFile in bakTxtFiles)
{
Console.WriteLine($"Processing BAK file: {Path.GetFileName(bakFile)}");
try
{
var bakPosts = ParsePostsFromFile(bakFile, allowedPrefixes);
Console.WriteLine($" Found {bakPosts.Count} post(s) in this file");
foreach (var (postId, bakData) in bakPosts)
{
totalPostsFound++;
var dbPost = DataAccess.GetPostByIdAnyBlog(postId);
if (dbPost == null)
{
postsNotFound++;
continue;
}
if (applyChanges)
{
bool updated = DataAccess.UpdatePostContentFields(dbPost.BlogName, dbPost.PostId, bakData);
if (updated)
{
postsUpdated++;
updateLog.Add($"Post ID: {postId} - Updated from {Path.GetFileName(bakFile)}");
}
}
else
{
var updateList = BuildDryRunDiff(bakData, dbPost);
if (updateList.Count > 0)
{
postsWithUpdates++;
correctionLog.Add($"\nPost ID: {postId}");
correctionLog.Add($" File: {Path.GetFileName(bakFile)}");
correctionLog.Add($" Fields to update:");
correctionLog.AddRange(updateList);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($" ERROR processing file: {ex.Message}");
}
}
if (applyChanges)
{
Console.WriteLine($"\n========== CORRECTION COMPLETE ==========");
Console.WriteLine($"Total posts found in BAK files: {totalPostsFound}");
Console.WriteLine($"Posts updated in database: {postsUpdated}");
Console.WriteLine($"Posts not found in database: {postsNotFound}");
string logPath = config["appSettings:PathCorrectionApplied"] ?? "correction_applied.txt";
try
{
var logLines = new List<string>
{
$"Correction Applied: {DateTime.Now:yyyy-MM-dd HH:mm:ss}",
$"Total posts found in BAK files: {totalPostsFound}",
$"Posts updated in database: {postsUpdated}",
$"Posts not found in database: {postsNotFound}",
"",
"Updated Posts:"
};
logLines.AddRange(updateLog);
File.WriteAllLines(logPath, logLines);
Console.WriteLine($"Update log saved to: {logPath}");
}
catch (Exception ex)
{
Console.WriteLine($"Error writing log file: {ex.Message}");
}
}
else
{
Console.WriteLine($"\n========== CORRECTION REPORT (DRY RUN) ==========");
Console.WriteLine($"Total posts found in BAK files: {totalPostsFound}");
Console.WriteLine($"Posts with multi-line field updates available: {postsWithUpdates}");
if (correctionLog.Count > 0)
{
string logPath = config["appSettings:PathCorrectionReport"] ?? "correction_report.txt";
try
{
File.WriteAllLines(logPath, correctionLog);
Console.WriteLine($"\nDetailed report saved to: {logPath}");
}
catch (Exception ex)
{
Console.WriteLine($"Error writing report file: {ex.Message}");
}
}
else
{
Console.WriteLine("\nNo multi-line field updates found.");
}
Console.WriteLine("\nDry-run complete. No database changes were made.");
Console.WriteLine("If updates look correct, re-run with `-correct -apply` to apply changes.");
}
return 0;
}
private static List<string> BuildDryRunDiff(Dictionary<string, string> bakData, TTPostRecord dbPost)
{
var updates = new List<string>();
foreach (var (fieldName, bakValue) in bakData)
{
if (string.IsNullOrWhiteSpace(bakValue)) continue;
string? currentValue = fieldName.ToLowerInvariant() switch
{
"reblog url" => dbPost.ReblogUrl,
"date" => dbPost.Date,
"has image" => dbPost.HasImage,
"post url" => dbPost.PostUrl,
"slug" => dbPost.Slug,
"reblog key" => dbPost.ReblogKey,
"reblog name" => dbPost.ReblogName,
"summary" => dbPost.Summary,
"quote" => dbPost.Quote,
"body" => dbPost.Body,
"tags" => dbPost.Tags,
"link" => dbPost.Link,
"photo url" => dbPost.PhotoUrl,
"photo caption" => dbPost.PhotoCaption,
"downloaded files" => dbPost.DownloadedFiles,
"audio caption" => dbPost.AudioCaption,
"question" => dbPost.Question,
"answer" => dbPost.Answer,
"title" => dbPost.Title,
_ => null
};
if (bakValue != currentValue && bakValue.Contains('\n'))
updates.Add($" {fieldName}: [MULTILINE]");
}
return updates;
}
private static Dictionary<string, Dictionary<string, string>> ParsePostsFromFile(string filePath, HashSet<string> allowedPrefixes)
{
var posts = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
var lines = File.ReadAllLines(filePath);
int lineIndex = 0;
string currentPostId = "";
var currentPostData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
while (lineIndex < lines.Length)
{
string line = lines[lineIndex];
string searchText = line.Length > 25 ? line.Substring(0, 25) : line;
int colonIndex = searchText.IndexOf(": ");
if (colonIndex > 0)
{
string prefix = line.Substring(0, colonIndex).Trim();
if (!string.IsNullOrWhiteSpace(prefix) && allowedPrefixes.Contains(prefix))
{
if (string.Equals(prefix, "Post ID", StringComparison.OrdinalIgnoreCase))
{
if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0)
posts[currentPostId] = new Dictionary<string, string>(currentPostData, StringComparer.OrdinalIgnoreCase);
currentPostId = line.Substring(colonIndex + 2).Trim();
currentPostData.Clear();
lineIndex++;
continue;
}
var valueLines = new List<string> { line.Substring(colonIndex + 2).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < lines.Length)
{
string nextLine = lines[nextLineIndex];
string nextSearch = nextLine.Length > 25 ? nextLine.Substring(0, 25) : nextLine;
int nextColon = nextSearch.IndexOf(": ");
if (nextColon > 0)
{
string nextPrefix = nextLine.Substring(0, nextColon).Trim();
if (!string.IsNullOrWhiteSpace(nextPrefix) && allowedPrefixes.Contains(nextPrefix))
break;
}
valueLines.Add(nextLine);
nextLineIndex++;
}
currentPostData[prefix] = string.Join("\n", valueLines);
lineIndex = nextLineIndex;
continue;
}
}
lineIndex++;
}
if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0)
posts[currentPostId] = new Dictionary<string, string>(currentPostData, StringComparer.OrdinalIgnoreCase);
return posts;
}
}
}
URLNotesGrabberCORE/DataAccess.cs
+513
View File
@@ -1786,6 +1786,519 @@ namespace URLNotesGrabberCORE
} }
} }
#endregion Updates #endregion Updates
#region TTFileHelper
// Idempotent migration: adds Blogs.TTFolderPath and Posts.PostType if missing.
// Mirrors the EnsureBlogsLikesColumnsExist pattern (PRAGMA table_info + ALTER TABLE ADD COLUMN).
public static void EnsureTTFileHelperColumnsExist(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
bool ttFolderPathExists = false;
using (SQLiteCommand command = new SQLiteCommand("PRAGMA table_info(Blogs);", connection))
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
string columnName = reader.GetString(1);
if (columnName.Equals("TTFolderPath", StringComparison.OrdinalIgnoreCase)) ttFolderPathExists = true;
}
}
if (!ttFolderPathExists)
{
using (SQLiteCommand cmd = new SQLiteCommand("ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;", connection))
cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added TTFolderPath column to Blogs table");
}
bool postTypeExists = false;
using (SQLiteCommand command = new SQLiteCommand("PRAGMA table_info(Posts);", connection))
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
string columnName = reader.GetString(1);
if (columnName.Equals("PostType", StringComparison.OrdinalIgnoreCase)) postTypeExists = true;
}
}
if (!postTypeExists)
{
using (SQLiteCommand cmd = new SQLiteCommand("ALTER TABLE Posts ADD COLUMN PostType TEXT;", connection))
cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added PostType column to Posts table");
}
}
catch (Exception ex)
{
Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}");
}
finally
{
connection.Close();
}
}
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
// On collision, only content columns + PostType + DateModified are updated;
// engagement columns (ByLikes, RootBlogName, RootURL, HasNotesGathered, NotFound,
// NotesGatheredDateTime) are preserved.
public static void UpsertPostFromTextFile(
string blogName,
string postID,
string? reblogURL,
string? postDate,
string? postURL,
string? slug,
string? reblogKey,
string? reblogName,
string? summary,
string? quote,
string? body,
string? tags,
string? link,
string? photoURL,
string? photoCaption,
string? downloadedFiles,
string? audioCaption,
string? question,
string? answer,
string? title,
string? postType,
bool hasImage,
string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
try { AddBlog(blogName, false, DBPath); } catch { }
SQLiteConnection connection;
bool ownsConnection;
lock (_importSessionLock)
{
if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
}
string now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
try
{
if (ownsConnection) connection.Open();
string insertSql = @"INSERT INTO Posts (
BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
DateModified, DateCreated, RootBlogName, RootURL,
HasImage, ByLikes
) VALUES (
@BlogName, @PostID, @reblogURL, @PostDate, @PostURL, @Slug,
@ReblogKey, @ReblogName, @Summary, @Quote, @Body, @Tags, @Link,
@PhotoURL, @PhotoCaption, @DownloadedFiles, @AudioCaption,
@Question, @Answer, @Title, @PostType,
@DateModified, @DateCreated, '.', '.', @HasImage, 0)";
int rowsInserted = 0;
try
{
using (var cmd = new SQLiteCommand(insertSql, connection))
{
cmd.Parameters.AddWithValue("@BlogName", blogName);
cmd.Parameters.AddWithValue("@PostID", postID);
cmd.Parameters.AddWithValue("@reblogURL", (object?)reblogURL ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PostDate", (object?)postDate ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PostURL", (object?)postURL ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Slug", (object?)slug ?? DBNull.Value);
cmd.Parameters.AddWithValue("@ReblogKey", (object?)reblogKey ?? DBNull.Value);
cmd.Parameters.AddWithValue("@ReblogName", (object?)reblogName ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Summary", (object?)summary ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Quote", (object?)quote ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Body", (object?)body ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Tags", (object?)tags ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Link", (object?)link ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PhotoURL", (object?)photoURL ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PhotoCaption", (object?)photoCaption ?? DBNull.Value);
cmd.Parameters.AddWithValue("@DownloadedFiles", (object?)downloadedFiles ?? DBNull.Value);
cmd.Parameters.AddWithValue("@AudioCaption", (object?)audioCaption ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Question", (object?)question ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Answer", (object?)answer ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Title", (object?)title ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PostType", (object?)postType ?? DBNull.Value);
cmd.Parameters.AddWithValue("@DateModified", now);
cmd.Parameters.AddWithValue("@DateCreated", now);
cmd.Parameters.AddWithValue("@HasImage", hasImage ? 1 : 0);
rowsInserted = cmd.ExecuteNonQuery();
}
}
catch (Exception insertEx)
{
string msg = insertEx.Message ?? string.Empty;
bool isUniqueViolation = msg.Contains("UNIQUE constraint failed");
if (!isUniqueViolation)
{
Console.WriteLine($"[UpsertPostFromTextFile] INSERT failed for {blogName}/{postID}: {msg}");
}
}
if (rowsInserted == 0)
{
string updateSql = @"UPDATE Posts SET
reblogURL = @reblogURL,
PostDate = @PostDate,
PostURL = @PostURL,
Slug = @Slug,
ReblogKey = @ReblogKey,
ReblogName = @ReblogName,
Summary = @Summary,
Quote = @Quote,
Body = @Body,
Tags = @Tags,
Link = @Link,
PhotoURL = @PhotoURL,
PhotoCaption = @PhotoCaption,
DownloadedFiles = @DownloadedFiles,
AudioCaption = @AudioCaption,
Question = @Question,
Answer = @Answer,
Title = @Title,
PostType = @PostType,
HasImage = @HasImage,
DateModified = @DateModified
WHERE BlogName = @BlogName AND PostID = @PostID";
using (var cmd = new SQLiteCommand(updateSql, connection))
{
cmd.Parameters.AddWithValue("@BlogName", blogName);
cmd.Parameters.AddWithValue("@PostID", postID);
cmd.Parameters.AddWithValue("@reblogURL", (object?)reblogURL ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PostDate", (object?)postDate ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PostURL", (object?)postURL ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Slug", (object?)slug ?? DBNull.Value);
cmd.Parameters.AddWithValue("@ReblogKey", (object?)reblogKey ?? DBNull.Value);
cmd.Parameters.AddWithValue("@ReblogName", (object?)reblogName ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Summary", (object?)summary ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Quote", (object?)quote ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Body", (object?)body ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Tags", (object?)tags ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Link", (object?)link ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PhotoURL", (object?)photoURL ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PhotoCaption", (object?)photoCaption ?? DBNull.Value);
cmd.Parameters.AddWithValue("@DownloadedFiles", (object?)downloadedFiles ?? DBNull.Value);
cmd.Parameters.AddWithValue("@AudioCaption", (object?)audioCaption ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Question", (object?)question ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Answer", (object?)answer ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Title", (object?)title ?? DBNull.Value);
cmd.Parameters.AddWithValue("@PostType", (object?)postType ?? DBNull.Value);
cmd.Parameters.AddWithValue("@HasImage", hasImage ? 1 : 0);
cmd.Parameters.AddWithValue("@DateModified", now);
cmd.ExecuteNonQuery();
}
}
}
finally
{
if (ownsConnection) connection.Close();
}
}
public static List<TTPostRecord> GetAllPostsForBlog(string blogName, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
var results = new List<TTPostRecord>();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE BlogName = @BlogName";
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@BlogName", blogName);
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
results.Add(new TTPostRecord
{
BlogName = SafeStr(reader, 0),
PostId = SafeStr(reader, 1),
ReblogUrl = SafeNullableStr(reader, 2),
Date = SafeNullableStr(reader, 3),
PostUrl = SafeNullableStr(reader, 4),
Slug = SafeNullableStr(reader, 5),
ReblogKey = SafeNullableStr(reader, 6),
ReblogName = SafeNullableStr(reader, 7),
Summary = SafeNullableStr(reader, 8),
Quote = SafeNullableStr(reader, 9),
Body = SafeNullableStr(reader, 10),
Tags = SafeNullableStr(reader, 11),
Link = SafeNullableStr(reader, 12),
PhotoUrl = SafeNullableStr(reader, 13),
PhotoCaption = SafeNullableStr(reader, 14),
DownloadedFiles = SafeNullableStr(reader, 15),
AudioCaption = SafeNullableStr(reader, 16),
Question = SafeNullableStr(reader, 17),
Answer = SafeNullableStr(reader, 18),
Title = SafeNullableStr(reader, 19),
PostType = SafeNullableStr(reader, 20),
HasImage = SafeNullableStr(reader, 21),
CreatedDate = SafeNullableStr(reader, 22),
ModifiedDate = SafeNullableStr(reader, 23)
});
}
return results;
}
public static TTPostRecord? GetPost(string blogName, string postId, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID";
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@BlogName", blogName);
cmd.Parameters.AddWithValue("@PostID", postId);
using var reader = cmd.ExecuteReader();
if (!reader.Read()) return null;
return new TTPostRecord
{
BlogName = SafeStr(reader, 0),
PostId = SafeStr(reader, 1),
ReblogUrl = SafeNullableStr(reader, 2),
Date = SafeNullableStr(reader, 3),
PostUrl = SafeNullableStr(reader, 4),
Slug = SafeNullableStr(reader, 5),
ReblogKey = SafeNullableStr(reader, 6),
ReblogName = SafeNullableStr(reader, 7),
Summary = SafeNullableStr(reader, 8),
Quote = SafeNullableStr(reader, 9),
Body = SafeNullableStr(reader, 10),
Tags = SafeNullableStr(reader, 11),
Link = SafeNullableStr(reader, 12),
PhotoUrl = SafeNullableStr(reader, 13),
PhotoCaption = SafeNullableStr(reader, 14),
DownloadedFiles = SafeNullableStr(reader, 15),
AudioCaption = SafeNullableStr(reader, 16),
Question = SafeNullableStr(reader, 17),
Answer = SafeNullableStr(reader, 18),
Title = SafeNullableStr(reader, 19),
PostType = SafeNullableStr(reader, 20),
HasImage = SafeNullableStr(reader, 21),
CreatedDate = SafeNullableStr(reader, 22),
ModifiedDate = SafeNullableStr(reader, 23)
};
}
// Find a post by PostID alone (matches ThreeTxtFileHelper's correction-mode
// lookup, which does not scope by blog name). Returns the first match if any.
public static TTPostRecord? GetPostByIdAnyBlog(string postId, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType,
HasImage, DateCreated, DateModified
FROM Posts WHERE PostID = @PostID LIMIT 1";
using var cmd = new SQLiteCommand(sql, connection);
cmd.Parameters.AddWithValue("@PostID", postId);
using var reader = cmd.ExecuteReader();
if (!reader.Read()) return null;
return new TTPostRecord
{
BlogName = SafeStr(reader, 0),
PostId = SafeStr(reader, 1),
ReblogUrl = SafeNullableStr(reader, 2),
Date = SafeNullableStr(reader, 3),
PostUrl = SafeNullableStr(reader, 4),
Slug = SafeNullableStr(reader, 5),
ReblogKey = SafeNullableStr(reader, 6),
ReblogName = SafeNullableStr(reader, 7),
Summary = SafeNullableStr(reader, 8),
Quote = SafeNullableStr(reader, 9),
Body = SafeNullableStr(reader, 10),
Tags = SafeNullableStr(reader, 11),
Link = SafeNullableStr(reader, 12),
PhotoUrl = SafeNullableStr(reader, 13),
PhotoCaption = SafeNullableStr(reader, 14),
DownloadedFiles = SafeNullableStr(reader, 15),
AudioCaption = SafeNullableStr(reader, 16),
Question = SafeNullableStr(reader, 17),
Answer = SafeNullableStr(reader, 18),
Title = SafeNullableStr(reader, 19),
PostType = SafeNullableStr(reader, 20),
HasImage = SafeNullableStr(reader, 21),
CreatedDate = SafeNullableStr(reader, 22),
ModifiedDate = SafeNullableStr(reader, 23)
};
}
public static void SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
try { AddBlog(blogName, false, DBPath); } catch { }
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using var cmd = new SQLiteCommand(
"UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name",
connection);
cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
cmd.Parameters.AddWithValue("@name", blogName);
cmd.ExecuteNonQuery();
}
// Partial UPDATE used by the correct-apply path. fieldsToUpdate maps
// ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty
// values pulled from a BAK file. Only those columns + DateModified are written;
// other content columns and all engagement columns are left intact.
// Returns true if a row was matched (and therefore updated).
public static bool UpdatePostContentFields(string blogName, string postId, IDictionary<string, string> fieldsToUpdate, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
var setClauses = new List<string>();
var parameters = new List<(string Name, object Value)>();
foreach (var kvp in fieldsToUpdate)
{
if (string.IsNullOrWhiteSpace(kvp.Value)) continue;
string? column = MapPrefixToColumn(kvp.Key);
if (column == null) continue;
string paramName = "@p" + parameters.Count;
setClauses.Add($"{column} = {paramName}");
parameters.Add((paramName, kvp.Value));
}
if (setClauses.Count == 0) return false;
setClauses.Add("DateModified = @DateModified");
parameters.Add(("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")));
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID";
using var cmd = new SQLiteCommand(sql, connection);
foreach (var (name, value) in parameters)
cmd.Parameters.AddWithValue(name, value);
cmd.Parameters.AddWithValue("@BlogName", blogName);
cmd.Parameters.AddWithValue("@PostID", postId);
return cmd.ExecuteNonQuery() > 0;
}
private static string? MapPrefixToColumn(string prefix)
{
return prefix.ToLowerInvariant() switch
{
"reblog url" => "reblogURL",
"date" => "PostDate",
"has image" => "HasImage",
"post url" => "PostURL",
"slug" => "Slug",
"reblog key" => "ReblogKey",
"reblog name" => "ReblogName",
"summary" => "Summary",
"quote" => "Quote",
"body" => "Body",
"tags" => "Tags",
"link" => "Link",
"photo url" => "PhotoURL",
"photo caption" => "PhotoCaption",
"downloaded files" => "DownloadedFiles",
"audio caption" => "AudioCaption",
"question" => "Question",
"answer" => "Answer",
"title" => "Title",
"post id" => null,
_ => null
};
}
public static List<(string BlogName, string? TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
var results = new List<(string, string?)>();
using var connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs", connection);
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
string name = reader.GetString(0);
string? path = reader.IsDBNull(1) ? null : reader.GetString(1);
results.Add((name, path));
}
return results;
}
private static string SafeStr(SQLiteDataReader reader, int ordinal)
{
return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty;
}
private static string? SafeNullableStr(SQLiteDataReader reader, int ordinal)
{
return reader.IsDBNull(ordinal) ? null : reader.GetValue(ordinal)?.ToString();
}
#endregion TTFileHelper
}
// Mirrors ThreeTxtFileHelper.PostData so the ported ingest/output/correct logic
// can read/write rows without an ORM. All content fields are nullable except keys.
public class TTPostRecord
{
public string BlogName { get; set; } = "";
public string PostId { get; set; } = "";
public string? ReblogUrl { get; set; }
public string? Date { get; set; }
public string? HasImage { get; set; }
public string? PostUrl { get; set; }
public string? Slug { get; set; }
public string? ReblogKey { get; set; }
public string? ReblogName { get; set; }
public string? Summary { get; set; }
public string? Quote { get; set; }
public string? Body { get; set; }
public string? Tags { get; set; }
public string? Link { get; set; }
public string? PhotoUrl { get; set; }
public string? PhotoCaption { get; set; }
public string? DownloadedFiles { get; set; }
public string? AudioCaption { get; set; }
public string? Question { get; set; }
public string? Answer { get; set; }
public string? Title { get; set; }
public string? PostType { get; set; }
public string? CreatedDate { get; set; }
public string? ModifiedDate { get; set; }
} }
public class ApiKeyConfig public class ApiKeyConfig
URLNotesGrabberCORE/IngestMode.cs
+199
View File
@@ -0,0 +1,199 @@
using System.Text.RegularExpressions;
using Microsoft.Extensions.Configuration;
namespace URLNotesGrabberCORE
{
// Port of ThreeTxtFileHelper RunIngestMode. Scans a root folder for .txt files,
// parses Tumblr-export fields (multi-line aware, prefix-driven), upserts each
// post into TL.db.Posts via DataAccess.UpsertPostFromTextFile.
public static class IngestMode
{
public static int Run(IConfiguration config, string[] args)
{
string? rootPath = args.Length > 0 ? args[0] : config["appSettings:PathTTRoot"];
if (string.IsNullOrWhiteSpace(rootPath))
rootPath = config["appSettings:PathInput"];
if (string.IsNullOrWhiteSpace(rootPath))
{
Console.WriteLine("Ingest: no root path provided. Set appSettings:PathTTRoot, appSettings:PathInput, or pass a path after -ingest.");
return 1;
}
if (!Directory.Exists(rootPath))
{
Console.WriteLine($"Directory not found: {rootPath}");
return 1;
}
DataAccess.EnsureTTFileHelperColumnsExist();
string prefixesPath = config["appSettings:PathPrefixes"] ?? "prefixes.txt";
if (!Path.IsPathRooted(prefixesPath))
prefixesPath = Path.Combine(AppContext.BaseDirectory, prefixesPath);
if (!File.Exists(prefixesPath))
{
Console.WriteLine($"Prefixes file not found: {prefixesPath}");
return 1;
}
var allowedPrefixes = new HashSet<string>(File.ReadLines(prefixesPath), StringComparer.OrdinalIgnoreCase);
Console.WriteLine($"Loaded {allowedPrefixes.Count} prefixes from {prefixesPath}");
Console.WriteLine($"========== Ingest Settings ==========");
Console.WriteLine($"Root path: {rootPath}");
Console.WriteLine($"=====================================");
var txtFiles = new List<string>();
try
{
var dirs = Directory.GetDirectories(rootPath, "*", SearchOption.AllDirectories);
Console.WriteLine($"Found {dirs.Length} directories under root.");
foreach (var dir in dirs)
{
try { txtFiles.AddRange(Directory.GetFiles(dir, "*.txt")); }
catch (Exception ex) { Console.WriteLine($" Skipping {dir}: {ex.Message}"); }
}
txtFiles.AddRange(Directory.GetFiles(rootPath, "*.txt"));
}
catch (Exception ex)
{
Console.WriteLine($"Error scanning root: {ex.Message}");
return 1;
}
Console.WriteLine($"Processing {txtFiles.Count} .txt file(s)...");
int filesProcessed = 0;
int postsTouched = 0;
try
{
DataAccess.EnableImportModePragmas();
DataAccess.BeginImportSession();
foreach (string file in txtFiles)
{
try
{
filesProcessed++;
if (filesProcessed % 50 == 0 || filesProcessed == 1)
Console.WriteLine($"[{filesProcessed}/{txtFiles.Count}] {Path.GetFileName(file)}");
string rawBlogName = Path.GetFileName(Path.GetDirectoryName(file) ?? "unknown");
string blogName = Regex.Replace(rawBlogName, @"_\d+$", "");
string postType = Path.GetFileNameWithoutExtension(file);
string currentPostId = "";
var currentPostData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
void Flush()
{
if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0)
{
UpsertPostFromParsedData(blogName, currentPostId, postType, currentPostData);
postsTouched++;
}
}
var lines = File.ReadAllLines(file);
int lineIndex = 0;
while (lineIndex < lines.Length)
{
string line = lines[lineIndex];
string searchText = line.Length > 25 ? line.Substring(0, 25) : line;
int colonIndex = searchText.IndexOf(": ");
if (colonIndex > 0)
{
string prefix = line.Substring(0, colonIndex).Trim();
if (!string.IsNullOrWhiteSpace(prefix) && allowedPrefixes.Contains(prefix))
{
if (string.Equals(prefix, "Post ID", StringComparison.OrdinalIgnoreCase))
{
Flush();
currentPostId = line.Substring(colonIndex + 2).Trim();
currentPostData.Clear();
lineIndex++;
continue;
}
var valueLines = new List<string> { line.Substring(colonIndex + 2).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < lines.Length)
{
string nextLine = lines[nextLineIndex];
string nextSearch = nextLine.Length > 25 ? nextLine.Substring(0, 25) : nextLine;
int nextColon = nextSearch.IndexOf(": ");
if (nextColon > 0)
{
string nextPrefix = nextLine.Substring(0, nextColon).Trim();
if (!string.IsNullOrWhiteSpace(nextPrefix) && allowedPrefixes.Contains(nextPrefix))
break;
}
valueLines.Add(nextLine);
nextLineIndex++;
}
currentPostData[prefix] = string.Join("\n", valueLines);
lineIndex = nextLineIndex;
continue;
}
}
lineIndex++;
}
Flush();
}
catch (Exception ex)
{
Console.WriteLine($" ERROR processing file {file}: {ex.Message}");
}
}
}
finally
{
DataAccess.EndImportSession();
DataAccess.RestoreImportModePragmas();
}
Console.WriteLine($"\nIngest complete. Files processed: {filesProcessed}. Posts touched: {postsTouched}.");
return 0;
}
private static void UpsertPostFromParsedData(string blogName, string postId, string postType, Dictionary<string, string> data)
{
string? G(string key) => data.TryGetValue(key, out var v) ? v : null;
string? hasImageStr = G("Has Image");
bool hasImage = !string.IsNullOrWhiteSpace(hasImageStr)
&& (hasImageStr.Equals("true", StringComparison.OrdinalIgnoreCase)
|| hasImageStr == "1"
|| hasImageStr.Equals("yes", StringComparison.OrdinalIgnoreCase));
DataAccess.UpsertPostFromTextFile(
blogName: blogName,
postID: postId,
reblogURL: G("reblog URL"),
postDate: G("Date"),
postURL: G("Post URL"),
slug: G("Slug"),
reblogKey: G("Reblog Key"),
reblogName: G("Reblog Name"),
summary: G("Summary"),
quote: G("Quote"),
body: G("Body"),
tags: G("Tags"),
link: G("Link"),
photoURL: G("Photo URL"),
photoCaption: G("Photo Caption"),
downloadedFiles: G("Downloaded Files"),
audioCaption: G("Audio Caption"),
question: G("Question"),
answer: G("Answer"),
title: G("Title"),
postType: postType,
hasImage: hasImage);
}
}
}
URLNotesGrabberCORE/LegacyPostsDbImporter.cs
+147
View File
@@ -0,0 +1,147 @@
using System.Data.SQLite;
namespace URLNotesGrabberCORE
{
// One-time migration: opens a legacy ThreeTxtFileHelper posts.db, copies its
// Blog + PostData rows into the merged TL.db via DataAccess.
// Conflict rule on (BlogName, PostId): ThreeTxtFileHelper wins on the 22 content
// columns + PostType + DateModified (handled inside UpsertPostFromTextFile).
// Engagement columns in TL.db (ByLikes, RootBlogName, RootURL, HasNotesGathered,
// NotFound, NotesGatheredDateTime) are preserved.
public static class LegacyPostsDbImporter
{
public static int Run(string legacyDbPath)
{
if (string.IsNullOrWhiteSpace(legacyDbPath))
{
Console.WriteLine("LegacyPostsDbImporter: path to legacy posts.db is required.");
return 1;
}
if (!File.Exists(legacyDbPath))
{
Console.WriteLine($"Legacy posts.db not found at: {legacyDbPath}");
return 1;
}
DataAccess.EnsureTTFileHelperColumnsExist();
Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}");
int blogsCopied = 0;
int postsUpserted = 0;
int errors = 0;
try
{
using var src = new SQLiteConnection("Data Source=" + legacyDbPath + ";Read Only=True;");
src.Open();
// 1) Copy Blogs (BlogName + TTFolderPath)
using (var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs", src))
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
string blogName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0);
string? ttFolderPath = reader.IsDBNull(1) ? null : reader.GetString(1);
if (string.IsNullOrWhiteSpace(blogName)) continue;
try
{
DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath);
blogsCopied++;
}
catch (Exception ex)
{
errors++;
Console.WriteLine($" Blog copy failed for '{blogName}': {ex.Message}");
}
}
}
Console.WriteLine($" Blogs copied: {blogsCopied}");
// 2) Copy Posts
try
{
DataAccess.EnableImportModePragmas();
DataAccess.BeginImportSession();
string sql = @"SELECT BlogName, PostId, ReblogUrl, Date, HasImage, PostUrl, Slug,
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
PhotoUrl, PhotoCaption, DownloadedFiles, AudioCaption,
Question, Answer, Title, PostType
FROM Posts";
using var cmd = new SQLiteCommand(sql, src);
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
try
{
string blogName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0);
string postId = reader.IsDBNull(1) ? string.Empty : reader.GetString(1);
if (string.IsNullOrWhiteSpace(blogName) || string.IsNullOrWhiteSpace(postId)) continue;
string? hasImageRaw = reader.IsDBNull(4) ? null : reader.GetValue(4)?.ToString();
bool hasImage = !string.IsNullOrWhiteSpace(hasImageRaw)
&& (hasImageRaw.Equals("true", StringComparison.OrdinalIgnoreCase)
|| hasImageRaw == "1"
|| hasImageRaw.Equals("yes", StringComparison.OrdinalIgnoreCase));
DataAccess.UpsertPostFromTextFile(
blogName: blogName,
postID: postId,
reblogURL: reader.IsDBNull(2) ? null : reader.GetString(2),
postDate: reader.IsDBNull(3) ? null : reader.GetString(3),
postURL: reader.IsDBNull(5) ? null : reader.GetString(5),
slug: reader.IsDBNull(6) ? null : reader.GetString(6),
reblogKey: reader.IsDBNull(7) ? null : reader.GetString(7),
reblogName: reader.IsDBNull(8) ? null : reader.GetString(8),
summary: reader.IsDBNull(9) ? null : reader.GetString(9),
quote: reader.IsDBNull(10) ? null : reader.GetString(10),
body: reader.IsDBNull(11) ? null : reader.GetString(11),
tags: reader.IsDBNull(12) ? null : reader.GetString(12),
link: reader.IsDBNull(13) ? null : reader.GetString(13),
photoURL: reader.IsDBNull(14) ? null : reader.GetString(14),
photoCaption: reader.IsDBNull(15) ? null : reader.GetString(15),
downloadedFiles: reader.IsDBNull(16) ? null : reader.GetString(16),
audioCaption: reader.IsDBNull(17) ? null : reader.GetString(17),
question: reader.IsDBNull(18) ? null : reader.GetString(18),
answer: reader.IsDBNull(19) ? null : reader.GetString(19),
title: reader.IsDBNull(20) ? null : reader.GetString(20),
postType: reader.IsDBNull(21) ? null : reader.GetString(21),
hasImage: hasImage);
postsUpserted++;
if (postsUpserted % 500 == 0)
Console.WriteLine($" ... {postsUpserted} posts upserted");
}
catch (Exception ex)
{
errors++;
if (errors < 20)
Console.WriteLine($" Post upsert error: {ex.Message}");
}
}
}
finally
{
DataAccess.EndImportSession();
DataAccess.RestoreImportModePragmas();
}
Console.WriteLine($" Posts upserted: {postsUpserted}");
}
catch (Exception ex)
{
Console.WriteLine($"Fatal error reading legacy posts.db: {ex.Message}");
return 1;
}
Console.WriteLine($"\n========== Legacy import summary ==========");
Console.WriteLine($"Blogs copied: {blogsCopied}");
Console.WriteLine($"Posts upserted: {postsUpserted}");
Console.WriteLine($"Errors: {errors}");
return errors == 0 ? 0 : 2;
}
}
}
URLNotesGrabberCORE/OutputMode.cs
+136
View File
@@ -0,0 +1,136 @@
using Microsoft.Extensions.Configuration;
namespace URLNotesGrabberCORE
{
// Port of ThreeTxtFileHelper RunOutputMode + WritePostToFile + RenameExistingTxtFilesToBak.
// For each Blog with a TTFolderPath, renames any existing .txt files in that folder to .bak,
// then writes one .txt per PostType containing all posts of that type (date-sorted, fixed
// field order). Reads from TL.db via DataAccess.GetAllPostsForBlog.
public static class OutputMode
{
public static int Run(IConfiguration config)
{
DataAccess.EnsureTTFileHelperColumnsExist();
var blogs = DataAccess.GetAllBlogsWithTTFolderPath();
Console.WriteLine($"Found {blogs.Count} blog(s) to process.");
foreach (var (blogName, ttFolderPath) in blogs)
{
Console.WriteLine($"\nProcessing blog: {blogName}");
if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath))
{
Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping.");
continue;
}
Console.WriteLine($" TTFolderPath: {ttFolderPath}");
try
{
foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak"))
File.Delete(bakFile);
}
catch (Exception ex)
{
Console.WriteLine($" Error deleting .bak files: {ex.Message}");
}
RenameExistingTxtFilesToBak(ttFolderPath);
var posts = DataAccess.GetAllPostsForBlog(blogName);
Console.WriteLine($" Found {posts.Count} post(s) for this blog.");
var grouped = posts.GroupBy(p => p.PostType ?? "Unknown");
foreach (var typeGroup in grouped)
{
string postType = typeGroup.Key ?? "Unknown";
string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt");
var ordered = typeGroup.OrderBy(p => p.Date).ToList();
Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt");
using var writer = new StreamWriter(outputFilePath, false, System.Text.Encoding.UTF8);
bool isFirst = true;
foreach (var post in ordered)
{
if (!isFirst)
{
writer.WriteLine();
writer.WriteLine();
}
WritePostToFile(writer, post);
isFirst = false;
}
}
}
Console.WriteLine("\nOutput mode complete.");
return 0;
}
private static void RenameExistingTxtFilesToBak(string folderPath)
{
try
{
foreach (var txtFile in Directory.GetFiles(folderPath, "*.txt"))
{
string bakPath = Path.ChangeExtension(txtFile, ".bak");
if (File.Exists(bakPath)) File.Delete(bakPath);
File.Move(txtFile, bakPath, overwrite: true);
}
}
catch (Exception ex)
{
Console.WriteLine($" Error renaming txt files to .bak: {ex.Message}");
}
}
private static void WritePostToFile(StreamWriter writer, TTPostRecord post)
{
var startColumns = new[] { "Post ID", "Date", "Post URL", "Slug", "Reblog Key", "Reblog URL", "Reblog Name", "Title", "Body" };
var endColumns = new[] { "Tags", "Downloaded Files" };
var columns = new Dictionary<string, string>();
if (!string.IsNullOrWhiteSpace(post.PostId)) columns["Post ID"] = post.PostId;
if (!string.IsNullOrWhiteSpace(post.Date)) columns["Date"] = post.Date!;
if (!string.IsNullOrWhiteSpace(post.PostUrl)) columns["Post URL"] = post.PostUrl!;
if (!string.IsNullOrWhiteSpace(post.Slug)) columns["Slug"] = post.Slug!;
if (!string.IsNullOrWhiteSpace(post.ReblogKey)) columns["Reblog Key"] = post.ReblogKey!;
if (!string.IsNullOrWhiteSpace(post.ReblogUrl)) columns["Reblog URL"] = post.ReblogUrl!;
if (!string.IsNullOrWhiteSpace(post.ReblogName)) columns["Reblog Name"] = post.ReblogName!;
if (!string.IsNullOrWhiteSpace(post.Title)) columns["Title"] = post.Title!;
if (!string.IsNullOrWhiteSpace(post.Body)) columns["Body"] = post.Body!;
if (!string.IsNullOrWhiteSpace(post.HasImage)) columns["Has Image"] = post.HasImage!;
if (!string.IsNullOrWhiteSpace(post.Summary)) columns["Summary"] = post.Summary!;
if (!string.IsNullOrWhiteSpace(post.Quote)) columns["Quote"] = post.Quote!;
if (!string.IsNullOrWhiteSpace(post.Link)) columns["Link"] = post.Link!;
if (!string.IsNullOrWhiteSpace(post.PhotoUrl)) columns["Photo URL"] = post.PhotoUrl!;
if (!string.IsNullOrWhiteSpace(post.PhotoCaption)) columns["Photo Caption"] = post.PhotoCaption!;
if (!string.IsNullOrWhiteSpace(post.AudioCaption)) columns["Audio Caption"] = post.AudioCaption!;
if (!string.IsNullOrWhiteSpace(post.Question)) columns["Question"] = post.Question!;
if (!string.IsNullOrWhiteSpace(post.Answer)) columns["Answer"] = post.Answer!;
if (!string.IsNullOrWhiteSpace(post.Tags)) columns["Tags"] = post.Tags!;
if (!string.IsNullOrWhiteSpace(post.DownloadedFiles)) columns["Downloaded Files"] = post.DownloadedFiles!;
foreach (var col in startColumns)
{
if (columns.ContainsKey(col))
{
writer.WriteLine($"{col}: {columns[col]}");
columns.Remove(col);
}
}
var remaining = columns.Keys.Where(k => !endColumns.Contains(k)).OrderBy(k => k).ToList();
foreach (var col in remaining)
writer.WriteLine($"{col}: {columns[col]}");
foreach (var col in endColumns)
{
if (columns.ContainsKey(col))
writer.WriteLine($"{col}: {columns[col]}");
}
}
}
}
URLNotesGrabberCORE/Program.cs
+48
View File
@@ -181,6 +181,18 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)"); Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
Console.WriteLine("-ingest [path]\t Ingest Tumblr .txt exports under path (or appSettings:PathTTRoot) into TL.db");
Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
Console.WriteLine("-updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from <root>\\Index and set Blogs.TTFolderPath");
Console.WriteLine("-importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");
break; break;
case "-parse": case "-parse":
@@ -309,6 +321,42 @@ namespace URLNotesGrabberCORE
DumpUrls(settings.GetValue<string>("PathOutputUrls")); DumpUrls(settings.GetValue<string>("PathOutputUrls"));
break; break;
case "-ingest":
IngestMode.Run(config, args.Skip(1).ToArray());
break;
case "-output":
OutputMode.Run(config);
break;
case "-correct":
{
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
var correctArgs = args.Skip(1)
.Where(a => !string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) && !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase))
.ToArray();
CorrectMode.Run(config, correctArgs, applyChanges);
break;
}
case "-updatepaths":
{
string rootPath = args.Length > 1 ? args[1] : (settings.GetValue<string>("PathTTRoot") ?? settings.GetValue<string>("PathInput") ?? string.Empty);
UpdateBlogPathsRunner.Run(rootPath);
break;
}
case "-importposts":
{
if (args.Length < 2)
{
Console.WriteLine("Usage: -importposts <path-to-legacy-posts.db>");
break;
}
LegacyPostsDbImporter.Run(args[1]);
break;
}
default: default:
Console.WriteLine("** Unknown Command ** " + args[0]); Console.WriteLine("** Unknown Command ** " + args[0]);
break; break;
URLNotesGrabberCORE/URLNotesGrabberCORE.csproj
+3
View File
@@ -31,6 +31,9 @@
<None Update="TL.db"> <None Update="TL.db">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None> </None>
<None Update="prefixes.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup> </ItemGroup>
</Project> </Project>
URLNotesGrabberCORE/UpdateBlogPathsRunner.cs
+70
View File
@@ -0,0 +1,70 @@
using System.Text.Json;
namespace URLNotesGrabberCORE
{
// Port of ThreeTxtFileHelper/UpdateBlogPaths.cs. Reads .tumblr / .tmblrpriv metadata
// files from a root\Index folder and populates Blogs.TTFolderPath in TL.db.
public static class UpdateBlogPathsRunner
{
public static int Run(string rootPath)
{
if (string.IsNullOrWhiteSpace(rootPath))
{
Console.WriteLine("UpdateBlogPaths: rootPath is required.");
return 1;
}
DataAccess.EnsureTTFileHelperColumnsExist();
string indexPath = Path.Combine(rootPath, "Index");
if (!Directory.Exists(indexPath))
{
Console.WriteLine($"Index folder not found at: {indexPath}");
return 1;
}
Console.WriteLine($"Scanning Index folder: {indexPath}");
var blogFiles = Directory.GetFiles(indexPath, "*.tumblr")
.Concat(Directory.GetFiles(indexPath, "*.tmblrpriv"))
.ToList();
Console.WriteLine($"Found {blogFiles.Count} blog metadata files");
int updatedCount = 0;
foreach (var blogFile in blogFiles)
{
try
{
string blogName = Path.GetFileNameWithoutExtension(blogFile);
string jsonContent = File.ReadAllText(blogFile);
using JsonDocument doc = JsonDocument.Parse(jsonContent);
JsonElement root = doc.RootElement;
if (root.TryGetProperty("FileDownloadLocation", out JsonElement locationElement))
{
string? fileDownloadLocation = locationElement.GetString();
if (!string.IsNullOrWhiteSpace(fileDownloadLocation))
{
DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation);
updatedCount++;
Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}");
}
}
else
{
Console.WriteLine($"No FileDownloadLocation found in {blogFile}");
}
}
catch (Exception ex)
{
Console.WriteLine($"Error processing {blogFile}: {ex.Message}");
}
}
Console.WriteLine($"\nUpdated {updatedCount} blogs with TTFolderPath");
return 0;
}
}
}
URLNotesGrabberCORE/appsettings.json
+6 -1
View File
@@ -12,7 +12,12 @@
"PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218", "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218",
"EnableFileLogging": false, "EnableFileLogging": false,
"LogTraversalRecordImports": false, "LogTraversalRecordImports": false,
"LikesRefreshCooldownDays": 7 "LikesRefreshCooldownDays": 7,
"PathTTRoot": "",
"PathTTBackup": "",
"PathPrefixes": "prefixes.txt",
"PathCorrectionReport": "correction_report.txt",
"PathCorrectionApplied": "correction_applied.txt"
}, },
"TumblrApi": { "TumblrApi": {
"ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3", "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",
URLNotesGrabberCORE/prefixes.txt
+20
View File
@@ -0,0 +1,20 @@
Post ID
reblog URL
Date
Has Image
Post URL
Slug
Reblog Key
Reblog Name
Summary
Quote
Body
Tags
Link
Photo URL
Photo Caption
Downloaded Files
Audio Caption
Question
Answer
Title