diff --git a/URLNotesGrabberCORE/CorrectMode.cs b/URLNotesGrabberCORE/CorrectMode.cs new file mode 100644 index 0000000..4381a4a --- /dev/null +++ b/URLNotesGrabberCORE/CorrectMode.cs @@ -0,0 +1,277 @@ +using Microsoft.Extensions.Configuration; + +namespace URLNotesGrabberCORE +{ + // Port of ThreeTxtFileHelper RunCorrectionMode (dry-run) and RunFullCorrectionMode (apply). + // Scans a BAK directory of .txt files, parses posts with multi-line field support, + // and either reports or applies content-column corrections to TL.db.Posts. + // Apply path only writes non-empty values (mirrors original ThreeTxtFileHelper semantics) + // and never touches engagement columns. + public static class CorrectMode + { + public static int Run(IConfiguration config, string[] args, bool applyChanges) + { + DataAccess.EnsureTTFileHelperColumnsExist(); + + // Resolve BAK path: explicit arg > appSettings:PathTTBackup > derive from PathTTRoot/PathInput + string? bakRootPath = args.Length > 0 ? args[0] : config["appSettings:PathTTBackup"]; + if (string.IsNullOrWhiteSpace(bakRootPath)) + { + string? root = config["appSettings:PathTTRoot"]; + if (string.IsNullOrWhiteSpace(root)) root = config["appSettings:PathInput"]; + if (!string.IsNullOrWhiteSpace(root)) + bakRootPath = root.TrimEnd('\\', '/') + "_BAK\\"; + } + + if (string.IsNullOrWhiteSpace(bakRootPath) || !Directory.Exists(bakRootPath)) + { + Console.WriteLine($"BAK directory not found: {bakRootPath}"); + return 1; + } + + Console.WriteLine($"BAK source path: {bakRootPath}"); + Console.WriteLine(applyChanges + ? "Correction mode: APPLY - non-empty fields from BAK overwrite DB columns\n" + : "Correction mode: Dry-run - reports multi-line field updates available\n"); + + string prefixesPath = config["appSettings:PathPrefixes"] ?? "prefixes.txt"; + if (!Path.IsPathRooted(prefixesPath)) + prefixesPath = Path.Combine(AppContext.BaseDirectory, prefixesPath); + if (!File.Exists(prefixesPath)) + { + Console.WriteLine($"Prefixes file not found: {prefixesPath}"); + return 1; + } + var allowedPrefixes = new HashSet(File.ReadLines(prefixesPath), StringComparer.OrdinalIgnoreCase); + + if (applyChanges) + { + Console.Write("WARNING: This will overwrite non-empty fields in matching posts from BAK files. Continue? (yes/no): "); + string? response = Console.ReadLine(); + if (string.IsNullOrWhiteSpace(response) || !response.Equals("yes", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine("Operation cancelled."); + return 0; + } + } + + var bakTxtFiles = new List(); + try + { + foreach (var dir in Directory.GetDirectories(bakRootPath, "*", SearchOption.AllDirectories)) + bakTxtFiles.AddRange(Directory.GetFiles(dir, "*.txt")); + bakTxtFiles.AddRange(Directory.GetFiles(bakRootPath, "*.txt")); + } + catch (Exception ex) + { + Console.WriteLine($"Error scanning BAK directory: {ex.Message}"); + return 1; + } + Console.WriteLine($"Found {bakTxtFiles.Count} file(s) in BAK directory\n"); + + int totalPostsFound = 0; + int postsWithUpdates = 0; + int postsUpdated = 0; + int postsNotFound = 0; + var correctionLog = new List(); + var updateLog = new List(); + + foreach (string bakFile in bakTxtFiles) + { + Console.WriteLine($"Processing BAK file: {Path.GetFileName(bakFile)}"); + try + { + var bakPosts = ParsePostsFromFile(bakFile, allowedPrefixes); + Console.WriteLine($" Found {bakPosts.Count} post(s) in this file"); + + foreach (var (postId, bakData) in bakPosts) + { + totalPostsFound++; + var dbPost = DataAccess.GetPostByIdAnyBlog(postId); + if (dbPost == null) + { + postsNotFound++; + continue; + } + + if (applyChanges) + { + bool updated = DataAccess.UpdatePostContentFields(dbPost.BlogName, dbPost.PostId, bakData); + if (updated) + { + postsUpdated++; + updateLog.Add($"Post ID: {postId} - Updated from {Path.GetFileName(bakFile)}"); + } + } + else + { + var updateList = BuildDryRunDiff(bakData, dbPost); + if (updateList.Count > 0) + { + postsWithUpdates++; + correctionLog.Add($"\nPost ID: {postId}"); + correctionLog.Add($" File: {Path.GetFileName(bakFile)}"); + correctionLog.Add($" Fields to update:"); + correctionLog.AddRange(updateList); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($" ERROR processing file: {ex.Message}"); + } + } + + if (applyChanges) + { + Console.WriteLine($"\n========== CORRECTION COMPLETE =========="); + Console.WriteLine($"Total posts found in BAK files: {totalPostsFound}"); + Console.WriteLine($"Posts updated in database: {postsUpdated}"); + Console.WriteLine($"Posts not found in database: {postsNotFound}"); + + string logPath = config["appSettings:PathCorrectionApplied"] ?? "correction_applied.txt"; + try + { + var logLines = new List + { + $"Correction Applied: {DateTime.Now:yyyy-MM-dd HH:mm:ss}", + $"Total posts found in BAK files: {totalPostsFound}", + $"Posts updated in database: {postsUpdated}", + $"Posts not found in database: {postsNotFound}", + "", + "Updated Posts:" + }; + logLines.AddRange(updateLog); + File.WriteAllLines(logPath, logLines); + Console.WriteLine($"Update log saved to: {logPath}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error writing log file: {ex.Message}"); + } + } + else + { + Console.WriteLine($"\n========== CORRECTION REPORT (DRY RUN) =========="); + Console.WriteLine($"Total posts found in BAK files: {totalPostsFound}"); + Console.WriteLine($"Posts with multi-line field updates available: {postsWithUpdates}"); + + if (correctionLog.Count > 0) + { + string logPath = config["appSettings:PathCorrectionReport"] ?? "correction_report.txt"; + try + { + File.WriteAllLines(logPath, correctionLog); + Console.WriteLine($"\nDetailed report saved to: {logPath}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error writing report file: {ex.Message}"); + } + } + else + { + Console.WriteLine("\nNo multi-line field updates found."); + } + + Console.WriteLine("\nDry-run complete. No database changes were made."); + Console.WriteLine("If updates look correct, re-run with `-correct -apply` to apply changes."); + } + return 0; + } + + private static List BuildDryRunDiff(Dictionary bakData, TTPostRecord dbPost) + { + var updates = new List(); + foreach (var (fieldName, bakValue) in bakData) + { + if (string.IsNullOrWhiteSpace(bakValue)) continue; + string? currentValue = fieldName.ToLowerInvariant() switch + { + "reblog url" => dbPost.ReblogUrl, + "date" => dbPost.Date, + "has image" => dbPost.HasImage, + "post url" => dbPost.PostUrl, + "slug" => dbPost.Slug, + "reblog key" => dbPost.ReblogKey, + "reblog name" => dbPost.ReblogName, + "summary" => dbPost.Summary, + "quote" => dbPost.Quote, + "body" => dbPost.Body, + "tags" => dbPost.Tags, + "link" => dbPost.Link, + "photo url" => dbPost.PhotoUrl, + "photo caption" => dbPost.PhotoCaption, + "downloaded files" => dbPost.DownloadedFiles, + "audio caption" => dbPost.AudioCaption, + "question" => dbPost.Question, + "answer" => dbPost.Answer, + "title" => dbPost.Title, + _ => null + }; + if (bakValue != currentValue && bakValue.Contains('\n')) + updates.Add($" {fieldName}: [MULTILINE]"); + } + return updates; + } + + private static Dictionary> ParsePostsFromFile(string filePath, HashSet allowedPrefixes) + { + var posts = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var lines = File.ReadAllLines(filePath); + int lineIndex = 0; + string currentPostId = ""; + var currentPostData = new Dictionary(StringComparer.OrdinalIgnoreCase); + + while (lineIndex < lines.Length) + { + string line = lines[lineIndex]; + string searchText = line.Length > 25 ? line.Substring(0, 25) : line; + int colonIndex = searchText.IndexOf(": "); + + if (colonIndex > 0) + { + string prefix = line.Substring(0, colonIndex).Trim(); + if (!string.IsNullOrWhiteSpace(prefix) && allowedPrefixes.Contains(prefix)) + { + if (string.Equals(prefix, "Post ID", StringComparison.OrdinalIgnoreCase)) + { + if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0) + posts[currentPostId] = new Dictionary(currentPostData, StringComparer.OrdinalIgnoreCase); + currentPostId = line.Substring(colonIndex + 2).Trim(); + currentPostData.Clear(); + lineIndex++; + continue; + } + + var valueLines = new List { line.Substring(colonIndex + 2).Trim() }; + int nextLineIndex = lineIndex + 1; + while (nextLineIndex < lines.Length) + { + string nextLine = lines[nextLineIndex]; + string nextSearch = nextLine.Length > 25 ? nextLine.Substring(0, 25) : nextLine; + int nextColon = nextSearch.IndexOf(": "); + if (nextColon > 0) + { + string nextPrefix = nextLine.Substring(0, nextColon).Trim(); + if (!string.IsNullOrWhiteSpace(nextPrefix) && allowedPrefixes.Contains(nextPrefix)) + break; + } + valueLines.Add(nextLine); + nextLineIndex++; + } + currentPostData[prefix] = string.Join("\n", valueLines); + lineIndex = nextLineIndex; + continue; + } + } + lineIndex++; + } + if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0) + posts[currentPostId] = new Dictionary(currentPostData, StringComparer.OrdinalIgnoreCase); + + return posts; + } + } +} diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 1f7f201..140b2bc 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -1786,6 +1786,519 @@ namespace URLNotesGrabberCORE } } #endregion Updates + + #region TTFileHelper + + // Idempotent migration: adds Blogs.TTFolderPath and Posts.PostType if missing. + // Mirrors the EnsureBlogsLikesColumnsExist pattern (PRAGMA table_info + ALTER TABLE ADD COLUMN). + public static void EnsureTTFileHelperColumnsExist(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + bool ttFolderPathExists = false; + using (SQLiteCommand command = new SQLiteCommand("PRAGMA table_info(Blogs);", connection)) + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + string columnName = reader.GetString(1); + if (columnName.Equals("TTFolderPath", StringComparison.OrdinalIgnoreCase)) ttFolderPathExists = true; + } + } + + if (!ttFolderPathExists) + { + using (SQLiteCommand cmd = new SQLiteCommand("ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;", connection)) + cmd.ExecuteNonQuery(); + Console.WriteLine("[Migration] Added TTFolderPath column to Blogs table"); + } + + bool postTypeExists = false; + using (SQLiteCommand command = new SQLiteCommand("PRAGMA table_info(Posts);", connection)) + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + string columnName = reader.GetString(1); + if (columnName.Equals("PostType", StringComparison.OrdinalIgnoreCase)) postTypeExists = true; + } + } + + if (!postTypeExists) + { + using (SQLiteCommand cmd = new SQLiteCommand("ALTER TABLE Posts ADD COLUMN PostType TEXT;", connection)) + cmd.ExecuteNonQuery(); + Console.WriteLine("[Migration] Added PostType column to Posts table"); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}"); + } + finally + { + connection.Close(); + } + } + + // INSERT-or-UPDATE for a post arriving from a Tumblr text-file export. + // On collision, only content columns + PostType + DateModified are updated; + // engagement columns (ByLikes, RootBlogName, RootURL, HasNotesGathered, NotFound, + // NotesGatheredDateTime) are preserved. + public static void UpsertPostFromTextFile( + string blogName, + string postID, + string? reblogURL, + string? postDate, + string? postURL, + string? slug, + string? reblogKey, + string? reblogName, + string? summary, + string? quote, + string? body, + string? tags, + string? link, + string? photoURL, + string? photoCaption, + string? downloadedFiles, + string? audioCaption, + string? question, + string? answer, + string? title, + string? postType, + bool hasImage, + string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + try { AddBlog(blogName, false, DBPath); } catch { } + + SQLiteConnection connection; + bool ownsConnection; + lock (_importSessionLock) + { + if (_importConnection != null) { connection = _importConnection; ownsConnection = false; } + else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; } + } + + string now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"); + + try + { + if (ownsConnection) connection.Open(); + + string insertSql = @"INSERT INTO Posts ( + BlogName, PostID, reblogURL, PostDate, PostURL, Slug, + ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, + PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, + Question, Answer, Title, PostType, + DateModified, DateCreated, RootBlogName, RootURL, + HasImage, ByLikes + ) VALUES ( + @BlogName, @PostID, @reblogURL, @PostDate, @PostURL, @Slug, + @ReblogKey, @ReblogName, @Summary, @Quote, @Body, @Tags, @Link, + @PhotoURL, @PhotoCaption, @DownloadedFiles, @AudioCaption, + @Question, @Answer, @Title, @PostType, + @DateModified, @DateCreated, '.', '.', @HasImage, 0)"; + + int rowsInserted = 0; + try + { + using (var cmd = new SQLiteCommand(insertSql, connection)) + { + cmd.Parameters.AddWithValue("@BlogName", blogName); + cmd.Parameters.AddWithValue("@PostID", postID); + cmd.Parameters.AddWithValue("@reblogURL", (object?)reblogURL ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PostDate", (object?)postDate ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PostURL", (object?)postURL ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Slug", (object?)slug ?? DBNull.Value); + cmd.Parameters.AddWithValue("@ReblogKey", (object?)reblogKey ?? DBNull.Value); + cmd.Parameters.AddWithValue("@ReblogName", (object?)reblogName ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Summary", (object?)summary ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Quote", (object?)quote ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Body", (object?)body ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Tags", (object?)tags ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Link", (object?)link ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PhotoURL", (object?)photoURL ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PhotoCaption", (object?)photoCaption ?? DBNull.Value); + cmd.Parameters.AddWithValue("@DownloadedFiles", (object?)downloadedFiles ?? DBNull.Value); + cmd.Parameters.AddWithValue("@AudioCaption", (object?)audioCaption ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Question", (object?)question ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Answer", (object?)answer ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Title", (object?)title ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PostType", (object?)postType ?? DBNull.Value); + cmd.Parameters.AddWithValue("@DateModified", now); + cmd.Parameters.AddWithValue("@DateCreated", now); + cmd.Parameters.AddWithValue("@HasImage", hasImage ? 1 : 0); + rowsInserted = cmd.ExecuteNonQuery(); + } + } + catch (Exception insertEx) + { + string msg = insertEx.Message ?? string.Empty; + bool isUniqueViolation = msg.Contains("UNIQUE constraint failed"); + if (!isUniqueViolation) + { + Console.WriteLine($"[UpsertPostFromTextFile] INSERT failed for {blogName}/{postID}: {msg}"); + } + } + + if (rowsInserted == 0) + { + string updateSql = @"UPDATE Posts SET + reblogURL = @reblogURL, + PostDate = @PostDate, + PostURL = @PostURL, + Slug = @Slug, + ReblogKey = @ReblogKey, + ReblogName = @ReblogName, + Summary = @Summary, + Quote = @Quote, + Body = @Body, + Tags = @Tags, + Link = @Link, + PhotoURL = @PhotoURL, + PhotoCaption = @PhotoCaption, + DownloadedFiles = @DownloadedFiles, + AudioCaption = @AudioCaption, + Question = @Question, + Answer = @Answer, + Title = @Title, + PostType = @PostType, + HasImage = @HasImage, + DateModified = @DateModified + WHERE BlogName = @BlogName AND PostID = @PostID"; + + using (var cmd = new SQLiteCommand(updateSql, connection)) + { + cmd.Parameters.AddWithValue("@BlogName", blogName); + cmd.Parameters.AddWithValue("@PostID", postID); + cmd.Parameters.AddWithValue("@reblogURL", (object?)reblogURL ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PostDate", (object?)postDate ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PostURL", (object?)postURL ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Slug", (object?)slug ?? DBNull.Value); + cmd.Parameters.AddWithValue("@ReblogKey", (object?)reblogKey ?? DBNull.Value); + cmd.Parameters.AddWithValue("@ReblogName", (object?)reblogName ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Summary", (object?)summary ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Quote", (object?)quote ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Body", (object?)body ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Tags", (object?)tags ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Link", (object?)link ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PhotoURL", (object?)photoURL ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PhotoCaption", (object?)photoCaption ?? DBNull.Value); + cmd.Parameters.AddWithValue("@DownloadedFiles", (object?)downloadedFiles ?? DBNull.Value); + cmd.Parameters.AddWithValue("@AudioCaption", (object?)audioCaption ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Question", (object?)question ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Answer", (object?)answer ?? DBNull.Value); + cmd.Parameters.AddWithValue("@Title", (object?)title ?? DBNull.Value); + cmd.Parameters.AddWithValue("@PostType", (object?)postType ?? DBNull.Value); + cmd.Parameters.AddWithValue("@HasImage", hasImage ? 1 : 0); + cmd.Parameters.AddWithValue("@DateModified", now); + cmd.ExecuteNonQuery(); + } + } + } + finally + { + if (ownsConnection) connection.Close(); + } + } + + public static List GetAllPostsForBlog(string blogName, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + var results = new List(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + + string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug, + ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, + PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, + Question, Answer, Title, PostType, + HasImage, DateCreated, DateModified + FROM Posts WHERE BlogName = @BlogName"; + using var cmd = new SQLiteCommand(sql, connection); + cmd.Parameters.AddWithValue("@BlogName", blogName); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + results.Add(new TTPostRecord + { + BlogName = SafeStr(reader, 0), + PostId = SafeStr(reader, 1), + ReblogUrl = SafeNullableStr(reader, 2), + Date = SafeNullableStr(reader, 3), + PostUrl = SafeNullableStr(reader, 4), + Slug = SafeNullableStr(reader, 5), + ReblogKey = SafeNullableStr(reader, 6), + ReblogName = SafeNullableStr(reader, 7), + Summary = SafeNullableStr(reader, 8), + Quote = SafeNullableStr(reader, 9), + Body = SafeNullableStr(reader, 10), + Tags = SafeNullableStr(reader, 11), + Link = SafeNullableStr(reader, 12), + PhotoUrl = SafeNullableStr(reader, 13), + PhotoCaption = SafeNullableStr(reader, 14), + DownloadedFiles = SafeNullableStr(reader, 15), + AudioCaption = SafeNullableStr(reader, 16), + Question = SafeNullableStr(reader, 17), + Answer = SafeNullableStr(reader, 18), + Title = SafeNullableStr(reader, 19), + PostType = SafeNullableStr(reader, 20), + HasImage = SafeNullableStr(reader, 21), + CreatedDate = SafeNullableStr(reader, 22), + ModifiedDate = SafeNullableStr(reader, 23) + }); + } + return results; + } + + public static TTPostRecord? GetPost(string blogName, string postId, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + + string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug, + ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, + PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, + Question, Answer, Title, PostType, + HasImage, DateCreated, DateModified + FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID"; + using var cmd = new SQLiteCommand(sql, connection); + cmd.Parameters.AddWithValue("@BlogName", blogName); + cmd.Parameters.AddWithValue("@PostID", postId); + using var reader = cmd.ExecuteReader(); + if (!reader.Read()) return null; + + return new TTPostRecord + { + BlogName = SafeStr(reader, 0), + PostId = SafeStr(reader, 1), + ReblogUrl = SafeNullableStr(reader, 2), + Date = SafeNullableStr(reader, 3), + PostUrl = SafeNullableStr(reader, 4), + Slug = SafeNullableStr(reader, 5), + ReblogKey = SafeNullableStr(reader, 6), + ReblogName = SafeNullableStr(reader, 7), + Summary = SafeNullableStr(reader, 8), + Quote = SafeNullableStr(reader, 9), + Body = SafeNullableStr(reader, 10), + Tags = SafeNullableStr(reader, 11), + Link = SafeNullableStr(reader, 12), + PhotoUrl = SafeNullableStr(reader, 13), + PhotoCaption = SafeNullableStr(reader, 14), + DownloadedFiles = SafeNullableStr(reader, 15), + AudioCaption = SafeNullableStr(reader, 16), + Question = SafeNullableStr(reader, 17), + Answer = SafeNullableStr(reader, 18), + Title = SafeNullableStr(reader, 19), + PostType = SafeNullableStr(reader, 20), + HasImage = SafeNullableStr(reader, 21), + CreatedDate = SafeNullableStr(reader, 22), + ModifiedDate = SafeNullableStr(reader, 23) + }; + } + + // Find a post by PostID alone (matches ThreeTxtFileHelper's correction-mode + // lookup, which does not scope by blog name). Returns the first match if any. + public static TTPostRecord? GetPostByIdAnyBlog(string postId, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + + string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug, + ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, + PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, + Question, Answer, Title, PostType, + HasImage, DateCreated, DateModified + FROM Posts WHERE PostID = @PostID LIMIT 1"; + using var cmd = new SQLiteCommand(sql, connection); + cmd.Parameters.AddWithValue("@PostID", postId); + using var reader = cmd.ExecuteReader(); + if (!reader.Read()) return null; + + return new TTPostRecord + { + BlogName = SafeStr(reader, 0), + PostId = SafeStr(reader, 1), + ReblogUrl = SafeNullableStr(reader, 2), + Date = SafeNullableStr(reader, 3), + PostUrl = SafeNullableStr(reader, 4), + Slug = SafeNullableStr(reader, 5), + ReblogKey = SafeNullableStr(reader, 6), + ReblogName = SafeNullableStr(reader, 7), + Summary = SafeNullableStr(reader, 8), + Quote = SafeNullableStr(reader, 9), + Body = SafeNullableStr(reader, 10), + Tags = SafeNullableStr(reader, 11), + Link = SafeNullableStr(reader, 12), + PhotoUrl = SafeNullableStr(reader, 13), + PhotoCaption = SafeNullableStr(reader, 14), + DownloadedFiles = SafeNullableStr(reader, 15), + AudioCaption = SafeNullableStr(reader, 16), + Question = SafeNullableStr(reader, 17), + Answer = SafeNullableStr(reader, 18), + Title = SafeNullableStr(reader, 19), + PostType = SafeNullableStr(reader, 20), + HasImage = SafeNullableStr(reader, 21), + CreatedDate = SafeNullableStr(reader, 22), + ModifiedDate = SafeNullableStr(reader, 23) + }; + } + + public static void SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + try { AddBlog(blogName, false, DBPath); } catch { } + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand( + "UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name", + connection); + cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value); + cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + cmd.Parameters.AddWithValue("@name", blogName); + cmd.ExecuteNonQuery(); + } + + // Partial UPDATE used by the correct-apply path. fieldsToUpdate maps + // ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty + // values pulled from a BAK file. Only those columns + DateModified are written; + // other content columns and all engagement columns are left intact. + // Returns true if a row was matched (and therefore updated). + public static bool UpdatePostContentFields(string blogName, string postId, IDictionary fieldsToUpdate, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + + var setClauses = new List(); + var parameters = new List<(string Name, object Value)>(); + + foreach (var kvp in fieldsToUpdate) + { + if (string.IsNullOrWhiteSpace(kvp.Value)) continue; + string? column = MapPrefixToColumn(kvp.Key); + if (column == null) continue; + string paramName = "@p" + parameters.Count; + setClauses.Add($"{column} = {paramName}"); + parameters.Add((paramName, kvp.Value)); + } + + if (setClauses.Count == 0) return false; + + setClauses.Add("DateModified = @DateModified"); + parameters.Add(("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"))); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + + string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID"; + using var cmd = new SQLiteCommand(sql, connection); + foreach (var (name, value) in parameters) + cmd.Parameters.AddWithValue(name, value); + cmd.Parameters.AddWithValue("@BlogName", blogName); + cmd.Parameters.AddWithValue("@PostID", postId); + + return cmd.ExecuteNonQuery() > 0; + } + + private static string? MapPrefixToColumn(string prefix) + { + return prefix.ToLowerInvariant() switch + { + "reblog url" => "reblogURL", + "date" => "PostDate", + "has image" => "HasImage", + "post url" => "PostURL", + "slug" => "Slug", + "reblog key" => "ReblogKey", + "reblog name" => "ReblogName", + "summary" => "Summary", + "quote" => "Quote", + "body" => "Body", + "tags" => "Tags", + "link" => "Link", + "photo url" => "PhotoURL", + "photo caption" => "PhotoCaption", + "downloaded files" => "DownloadedFiles", + "audio caption" => "AudioCaption", + "question" => "Question", + "answer" => "Answer", + "title" => "Title", + "post id" => null, + _ => null + }; + } + + public static List<(string BlogName, string? TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + var results = new List<(string, string?)>(); + + using var connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + using var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs", connection); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + string name = reader.GetString(0); + string? path = reader.IsDBNull(1) ? null : reader.GetString(1); + results.Add((name, path)); + } + return results; + } + + private static string SafeStr(SQLiteDataReader reader, int ordinal) + { + return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty; + } + + private static string? SafeNullableStr(SQLiteDataReader reader, int ordinal) + { + return reader.IsDBNull(ordinal) ? null : reader.GetValue(ordinal)?.ToString(); + } + + #endregion TTFileHelper + } + + // Mirrors ThreeTxtFileHelper.PostData so the ported ingest/output/correct logic + // can read/write rows without an ORM. All content fields are nullable except keys. + public class TTPostRecord + { + public string BlogName { get; set; } = ""; + public string PostId { get; set; } = ""; + public string? ReblogUrl { get; set; } + public string? Date { get; set; } + public string? HasImage { get; set; } + public string? PostUrl { get; set; } + public string? Slug { get; set; } + public string? ReblogKey { get; set; } + public string? ReblogName { get; set; } + public string? Summary { get; set; } + public string? Quote { get; set; } + public string? Body { get; set; } + public string? Tags { get; set; } + public string? Link { get; set; } + public string? PhotoUrl { get; set; } + public string? PhotoCaption { get; set; } + public string? DownloadedFiles { get; set; } + public string? AudioCaption { get; set; } + public string? Question { get; set; } + public string? Answer { get; set; } + public string? Title { get; set; } + public string? PostType { get; set; } + public string? CreatedDate { get; set; } + public string? ModifiedDate { get; set; } } public class ApiKeyConfig diff --git a/URLNotesGrabberCORE/IngestMode.cs b/URLNotesGrabberCORE/IngestMode.cs new file mode 100644 index 0000000..ae999d2 --- /dev/null +++ b/URLNotesGrabberCORE/IngestMode.cs @@ -0,0 +1,199 @@ +using System.Text.RegularExpressions; +using Microsoft.Extensions.Configuration; + +namespace URLNotesGrabberCORE +{ + // Port of ThreeTxtFileHelper RunIngestMode. Scans a root folder for .txt files, + // parses Tumblr-export fields (multi-line aware, prefix-driven), upserts each + // post into TL.db.Posts via DataAccess.UpsertPostFromTextFile. + public static class IngestMode + { + public static int Run(IConfiguration config, string[] args) + { + string? rootPath = args.Length > 0 ? args[0] : config["appSettings:PathTTRoot"]; + if (string.IsNullOrWhiteSpace(rootPath)) + rootPath = config["appSettings:PathInput"]; + + if (string.IsNullOrWhiteSpace(rootPath)) + { + Console.WriteLine("Ingest: no root path provided. Set appSettings:PathTTRoot, appSettings:PathInput, or pass a path after -ingest."); + return 1; + } + + if (!Directory.Exists(rootPath)) + { + Console.WriteLine($"Directory not found: {rootPath}"); + return 1; + } + + DataAccess.EnsureTTFileHelperColumnsExist(); + + string prefixesPath = config["appSettings:PathPrefixes"] ?? "prefixes.txt"; + if (!Path.IsPathRooted(prefixesPath)) + prefixesPath = Path.Combine(AppContext.BaseDirectory, prefixesPath); + + if (!File.Exists(prefixesPath)) + { + Console.WriteLine($"Prefixes file not found: {prefixesPath}"); + return 1; + } + + var allowedPrefixes = new HashSet(File.ReadLines(prefixesPath), StringComparer.OrdinalIgnoreCase); + Console.WriteLine($"Loaded {allowedPrefixes.Count} prefixes from {prefixesPath}"); + + Console.WriteLine($"========== Ingest Settings =========="); + Console.WriteLine($"Root path: {rootPath}"); + Console.WriteLine($"====================================="); + + var txtFiles = new List(); + try + { + var dirs = Directory.GetDirectories(rootPath, "*", SearchOption.AllDirectories); + Console.WriteLine($"Found {dirs.Length} directories under root."); + foreach (var dir in dirs) + { + try { txtFiles.AddRange(Directory.GetFiles(dir, "*.txt")); } + catch (Exception ex) { Console.WriteLine($" Skipping {dir}: {ex.Message}"); } + } + txtFiles.AddRange(Directory.GetFiles(rootPath, "*.txt")); + } + catch (Exception ex) + { + Console.WriteLine($"Error scanning root: {ex.Message}"); + return 1; + } + + Console.WriteLine($"Processing {txtFiles.Count} .txt file(s)..."); + + int filesProcessed = 0; + int postsTouched = 0; + + try + { + DataAccess.EnableImportModePragmas(); + DataAccess.BeginImportSession(); + + foreach (string file in txtFiles) + { + try + { + filesProcessed++; + if (filesProcessed % 50 == 0 || filesProcessed == 1) + Console.WriteLine($"[{filesProcessed}/{txtFiles.Count}] {Path.GetFileName(file)}"); + + string rawBlogName = Path.GetFileName(Path.GetDirectoryName(file) ?? "unknown"); + string blogName = Regex.Replace(rawBlogName, @"_\d+$", ""); + string postType = Path.GetFileNameWithoutExtension(file); + + string currentPostId = ""; + var currentPostData = new Dictionary(StringComparer.OrdinalIgnoreCase); + + void Flush() + { + if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0) + { + UpsertPostFromParsedData(blogName, currentPostId, postType, currentPostData); + postsTouched++; + } + } + + var lines = File.ReadAllLines(file); + int lineIndex = 0; + while (lineIndex < lines.Length) + { + string line = lines[lineIndex]; + string searchText = line.Length > 25 ? line.Substring(0, 25) : line; + int colonIndex = searchText.IndexOf(": "); + + if (colonIndex > 0) + { + string prefix = line.Substring(0, colonIndex).Trim(); + if (!string.IsNullOrWhiteSpace(prefix) && allowedPrefixes.Contains(prefix)) + { + if (string.Equals(prefix, "Post ID", StringComparison.OrdinalIgnoreCase)) + { + Flush(); + currentPostId = line.Substring(colonIndex + 2).Trim(); + currentPostData.Clear(); + lineIndex++; + continue; + } + + var valueLines = new List { line.Substring(colonIndex + 2).Trim() }; + int nextLineIndex = lineIndex + 1; + while (nextLineIndex < lines.Length) + { + string nextLine = lines[nextLineIndex]; + string nextSearch = nextLine.Length > 25 ? nextLine.Substring(0, 25) : nextLine; + int nextColon = nextSearch.IndexOf(": "); + if (nextColon > 0) + { + string nextPrefix = nextLine.Substring(0, nextColon).Trim(); + if (!string.IsNullOrWhiteSpace(nextPrefix) && allowedPrefixes.Contains(nextPrefix)) + break; + } + valueLines.Add(nextLine); + nextLineIndex++; + } + + currentPostData[prefix] = string.Join("\n", valueLines); + lineIndex = nextLineIndex; + continue; + } + } + lineIndex++; + } + + Flush(); + } + catch (Exception ex) + { + Console.WriteLine($" ERROR processing file {file}: {ex.Message}"); + } + } + } + finally + { + DataAccess.EndImportSession(); + DataAccess.RestoreImportModePragmas(); + } + + Console.WriteLine($"\nIngest complete. Files processed: {filesProcessed}. Posts touched: {postsTouched}."); + return 0; + } + + private static void UpsertPostFromParsedData(string blogName, string postId, string postType, Dictionary data) + { + string? G(string key) => data.TryGetValue(key, out var v) ? v : null; + string? hasImageStr = G("Has Image"); + bool hasImage = !string.IsNullOrWhiteSpace(hasImageStr) + && (hasImageStr.Equals("true", StringComparison.OrdinalIgnoreCase) + || hasImageStr == "1" + || hasImageStr.Equals("yes", StringComparison.OrdinalIgnoreCase)); + + DataAccess.UpsertPostFromTextFile( + blogName: blogName, + postID: postId, + reblogURL: G("reblog URL"), + postDate: G("Date"), + postURL: G("Post URL"), + slug: G("Slug"), + reblogKey: G("Reblog Key"), + reblogName: G("Reblog Name"), + summary: G("Summary"), + quote: G("Quote"), + body: G("Body"), + tags: G("Tags"), + link: G("Link"), + photoURL: G("Photo URL"), + photoCaption: G("Photo Caption"), + downloadedFiles: G("Downloaded Files"), + audioCaption: G("Audio Caption"), + question: G("Question"), + answer: G("Answer"), + title: G("Title"), + postType: postType, + hasImage: hasImage); + } + } +} diff --git a/URLNotesGrabberCORE/LegacyPostsDbImporter.cs b/URLNotesGrabberCORE/LegacyPostsDbImporter.cs new file mode 100644 index 0000000..a834652 --- /dev/null +++ b/URLNotesGrabberCORE/LegacyPostsDbImporter.cs @@ -0,0 +1,147 @@ +using System.Data.SQLite; + +namespace URLNotesGrabberCORE +{ + // One-time migration: opens a legacy ThreeTxtFileHelper posts.db, copies its + // Blog + PostData rows into the merged TL.db via DataAccess. + // Conflict rule on (BlogName, PostId): ThreeTxtFileHelper wins on the 22 content + // columns + PostType + DateModified (handled inside UpsertPostFromTextFile). + // Engagement columns in TL.db (ByLikes, RootBlogName, RootURL, HasNotesGathered, + // NotFound, NotesGatheredDateTime) are preserved. + public static class LegacyPostsDbImporter + { + public static int Run(string legacyDbPath) + { + if (string.IsNullOrWhiteSpace(legacyDbPath)) + { + Console.WriteLine("LegacyPostsDbImporter: path to legacy posts.db is required."); + return 1; + } + + if (!File.Exists(legacyDbPath)) + { + Console.WriteLine($"Legacy posts.db not found at: {legacyDbPath}"); + return 1; + } + + DataAccess.EnsureTTFileHelperColumnsExist(); + + Console.WriteLine($"Reading legacy posts.db: {legacyDbPath}"); + + int blogsCopied = 0; + int postsUpserted = 0; + int errors = 0; + + try + { + using var src = new SQLiteConnection("Data Source=" + legacyDbPath + ";Read Only=True;"); + src.Open(); + + // 1) Copy Blogs (BlogName + TTFolderPath) + using (var cmd = new SQLiteCommand("SELECT BlogName, TTFolderPath FROM Blogs", src)) + using (var reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + string blogName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0); + string? ttFolderPath = reader.IsDBNull(1) ? null : reader.GetString(1); + if (string.IsNullOrWhiteSpace(blogName)) continue; + try + { + DataAccess.SetBlogTTFolderPath(blogName, ttFolderPath); + blogsCopied++; + } + catch (Exception ex) + { + errors++; + Console.WriteLine($" Blog copy failed for '{blogName}': {ex.Message}"); + } + } + } + Console.WriteLine($" Blogs copied: {blogsCopied}"); + + // 2) Copy Posts + try + { + DataAccess.EnableImportModePragmas(); + DataAccess.BeginImportSession(); + + string sql = @"SELECT BlogName, PostId, ReblogUrl, Date, HasImage, PostUrl, Slug, + ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, + PhotoUrl, PhotoCaption, DownloadedFiles, AudioCaption, + Question, Answer, Title, PostType + FROM Posts"; + + using var cmd = new SQLiteCommand(sql, src); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + try + { + string blogName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0); + string postId = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); + if (string.IsNullOrWhiteSpace(blogName) || string.IsNullOrWhiteSpace(postId)) continue; + + string? hasImageRaw = reader.IsDBNull(4) ? null : reader.GetValue(4)?.ToString(); + bool hasImage = !string.IsNullOrWhiteSpace(hasImageRaw) + && (hasImageRaw.Equals("true", StringComparison.OrdinalIgnoreCase) + || hasImageRaw == "1" + || hasImageRaw.Equals("yes", StringComparison.OrdinalIgnoreCase)); + + DataAccess.UpsertPostFromTextFile( + blogName: blogName, + postID: postId, + reblogURL: reader.IsDBNull(2) ? null : reader.GetString(2), + postDate: reader.IsDBNull(3) ? null : reader.GetString(3), + postURL: reader.IsDBNull(5) ? null : reader.GetString(5), + slug: reader.IsDBNull(6) ? null : reader.GetString(6), + reblogKey: reader.IsDBNull(7) ? null : reader.GetString(7), + reblogName: reader.IsDBNull(8) ? null : reader.GetString(8), + summary: reader.IsDBNull(9) ? null : reader.GetString(9), + quote: reader.IsDBNull(10) ? null : reader.GetString(10), + body: reader.IsDBNull(11) ? null : reader.GetString(11), + tags: reader.IsDBNull(12) ? null : reader.GetString(12), + link: reader.IsDBNull(13) ? null : reader.GetString(13), + photoURL: reader.IsDBNull(14) ? null : reader.GetString(14), + photoCaption: reader.IsDBNull(15) ? null : reader.GetString(15), + downloadedFiles: reader.IsDBNull(16) ? null : reader.GetString(16), + audioCaption: reader.IsDBNull(17) ? null : reader.GetString(17), + question: reader.IsDBNull(18) ? null : reader.GetString(18), + answer: reader.IsDBNull(19) ? null : reader.GetString(19), + title: reader.IsDBNull(20) ? null : reader.GetString(20), + postType: reader.IsDBNull(21) ? null : reader.GetString(21), + hasImage: hasImage); + postsUpserted++; + if (postsUpserted % 500 == 0) + Console.WriteLine($" ... {postsUpserted} posts upserted"); + } + catch (Exception ex) + { + errors++; + if (errors < 20) + Console.WriteLine($" Post upsert error: {ex.Message}"); + } + } + } + finally + { + DataAccess.EndImportSession(); + DataAccess.RestoreImportModePragmas(); + } + + Console.WriteLine($" Posts upserted: {postsUpserted}"); + } + catch (Exception ex) + { + Console.WriteLine($"Fatal error reading legacy posts.db: {ex.Message}"); + return 1; + } + + Console.WriteLine($"\n========== Legacy import summary =========="); + Console.WriteLine($"Blogs copied: {blogsCopied}"); + Console.WriteLine($"Posts upserted: {postsUpserted}"); + Console.WriteLine($"Errors: {errors}"); + return errors == 0 ? 0 : 2; + } + } +} diff --git a/URLNotesGrabberCORE/OutputMode.cs b/URLNotesGrabberCORE/OutputMode.cs new file mode 100644 index 0000000..c060880 --- /dev/null +++ b/URLNotesGrabberCORE/OutputMode.cs @@ -0,0 +1,136 @@ +using Microsoft.Extensions.Configuration; + +namespace URLNotesGrabberCORE +{ + // Port of ThreeTxtFileHelper RunOutputMode + WritePostToFile + RenameExistingTxtFilesToBak. + // For each Blog with a TTFolderPath, renames any existing .txt files in that folder to .bak, + // then writes one .txt per PostType containing all posts of that type (date-sorted, fixed + // field order). Reads from TL.db via DataAccess.GetAllPostsForBlog. + public static class OutputMode + { + public static int Run(IConfiguration config) + { + DataAccess.EnsureTTFileHelperColumnsExist(); + + var blogs = DataAccess.GetAllBlogsWithTTFolderPath(); + Console.WriteLine($"Found {blogs.Count} blog(s) to process."); + + foreach (var (blogName, ttFolderPath) in blogs) + { + Console.WriteLine($"\nProcessing blog: {blogName}"); + + if (string.IsNullOrWhiteSpace(ttFolderPath) || !Directory.Exists(ttFolderPath)) + { + Console.WriteLine($" TTFolderPath does not exist or is not set. Skipping."); + continue; + } + + Console.WriteLine($" TTFolderPath: {ttFolderPath}"); + + try + { + foreach (var bakFile in Directory.GetFiles(ttFolderPath, "*.bak")) + File.Delete(bakFile); + } + catch (Exception ex) + { + Console.WriteLine($" Error deleting .bak files: {ex.Message}"); + } + + RenameExistingTxtFilesToBak(ttFolderPath); + + var posts = DataAccess.GetAllPostsForBlog(blogName); + Console.WriteLine($" Found {posts.Count} post(s) for this blog."); + + var grouped = posts.GroupBy(p => p.PostType ?? "Unknown"); + foreach (var typeGroup in grouped) + { + string postType = typeGroup.Key ?? "Unknown"; + string outputFilePath = Path.Combine(ttFolderPath, $"{postType}.txt"); + var ordered = typeGroup.OrderBy(p => p.Date).ToList(); + Console.WriteLine($" Writing {ordered.Count} post(s) to {postType}.txt"); + + using var writer = new StreamWriter(outputFilePath, false, System.Text.Encoding.UTF8); + bool isFirst = true; + foreach (var post in ordered) + { + if (!isFirst) + { + writer.WriteLine(); + writer.WriteLine(); + } + WritePostToFile(writer, post); + isFirst = false; + } + } + } + + Console.WriteLine("\nOutput mode complete."); + return 0; + } + + private static void RenameExistingTxtFilesToBak(string folderPath) + { + try + { + foreach (var txtFile in Directory.GetFiles(folderPath, "*.txt")) + { + string bakPath = Path.ChangeExtension(txtFile, ".bak"); + if (File.Exists(bakPath)) File.Delete(bakPath); + File.Move(txtFile, bakPath, overwrite: true); + } + } + catch (Exception ex) + { + Console.WriteLine($" Error renaming txt files to .bak: {ex.Message}"); + } + } + + private static void WritePostToFile(StreamWriter writer, TTPostRecord post) + { + var startColumns = new[] { "Post ID", "Date", "Post URL", "Slug", "Reblog Key", "Reblog URL", "Reblog Name", "Title", "Body" }; + var endColumns = new[] { "Tags", "Downloaded Files" }; + + var columns = new Dictionary(); + if (!string.IsNullOrWhiteSpace(post.PostId)) columns["Post ID"] = post.PostId; + if (!string.IsNullOrWhiteSpace(post.Date)) columns["Date"] = post.Date!; + if (!string.IsNullOrWhiteSpace(post.PostUrl)) columns["Post URL"] = post.PostUrl!; + if (!string.IsNullOrWhiteSpace(post.Slug)) columns["Slug"] = post.Slug!; + if (!string.IsNullOrWhiteSpace(post.ReblogKey)) columns["Reblog Key"] = post.ReblogKey!; + if (!string.IsNullOrWhiteSpace(post.ReblogUrl)) columns["Reblog URL"] = post.ReblogUrl!; + if (!string.IsNullOrWhiteSpace(post.ReblogName)) columns["Reblog Name"] = post.ReblogName!; + if (!string.IsNullOrWhiteSpace(post.Title)) columns["Title"] = post.Title!; + if (!string.IsNullOrWhiteSpace(post.Body)) columns["Body"] = post.Body!; + if (!string.IsNullOrWhiteSpace(post.HasImage)) columns["Has Image"] = post.HasImage!; + if (!string.IsNullOrWhiteSpace(post.Summary)) columns["Summary"] = post.Summary!; + if (!string.IsNullOrWhiteSpace(post.Quote)) columns["Quote"] = post.Quote!; + if (!string.IsNullOrWhiteSpace(post.Link)) columns["Link"] = post.Link!; + if (!string.IsNullOrWhiteSpace(post.PhotoUrl)) columns["Photo URL"] = post.PhotoUrl!; + if (!string.IsNullOrWhiteSpace(post.PhotoCaption)) columns["Photo Caption"] = post.PhotoCaption!; + if (!string.IsNullOrWhiteSpace(post.AudioCaption)) columns["Audio Caption"] = post.AudioCaption!; + if (!string.IsNullOrWhiteSpace(post.Question)) columns["Question"] = post.Question!; + if (!string.IsNullOrWhiteSpace(post.Answer)) columns["Answer"] = post.Answer!; + if (!string.IsNullOrWhiteSpace(post.Tags)) columns["Tags"] = post.Tags!; + if (!string.IsNullOrWhiteSpace(post.DownloadedFiles)) columns["Downloaded Files"] = post.DownloadedFiles!; + + foreach (var col in startColumns) + { + if (columns.ContainsKey(col)) + { + writer.WriteLine($"{col}: {columns[col]}"); + columns.Remove(col); + } + } + + var remaining = columns.Keys.Where(k => !endColumns.Contains(k)).OrderBy(k => k).ToList(); + foreach (var col in remaining) + writer.WriteLine($"{col}: {columns[col]}"); + + foreach (var col in endColumns) + { + if (columns.ContainsKey(col)) + writer.WriteLine($"{col}: {columns[col]}"); + } + } + } +} diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 7ef231c..3211640 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -181,6 +181,18 @@ namespace URLNotesGrabberCORE Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)"); + Console.WriteLine("-ingest [path]\t Ingest Tumblr .txt exports under path (or appSettings:PathTTRoot) into TL.db"); + + Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath"); + + Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory"); + + Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)"); + + Console.WriteLine("-updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from \\Index and set Blogs.TTFolderPath"); + + Console.WriteLine("-importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db"); + break; case "-parse": @@ -309,6 +321,42 @@ namespace URLNotesGrabberCORE DumpUrls(settings.GetValue("PathOutputUrls")); break; + case "-ingest": + IngestMode.Run(config, args.Skip(1).ToArray()); + break; + + case "-output": + OutputMode.Run(config); + break; + + case "-correct": + { + bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase)); + var correctArgs = args.Skip(1) + .Where(a => !string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) && !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + CorrectMode.Run(config, correctArgs, applyChanges); + break; + } + + case "-updatepaths": + { + string rootPath = args.Length > 1 ? args[1] : (settings.GetValue("PathTTRoot") ?? settings.GetValue("PathInput") ?? string.Empty); + UpdateBlogPathsRunner.Run(rootPath); + break; + } + + case "-importposts": + { + if (args.Length < 2) + { + Console.WriteLine("Usage: -importposts "); + break; + } + LegacyPostsDbImporter.Run(args[1]); + break; + } + default: Console.WriteLine("** Unknown Command ** " + args[0]); break; diff --git a/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj b/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj index 6bfef33..d636f2e 100644 --- a/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj +++ b/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj @@ -31,6 +31,9 @@ PreserveNewest + + PreserveNewest + diff --git a/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs b/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs new file mode 100644 index 0000000..47e6548 --- /dev/null +++ b/URLNotesGrabberCORE/UpdateBlogPathsRunner.cs @@ -0,0 +1,70 @@ +using System.Text.Json; + +namespace URLNotesGrabberCORE +{ + // Port of ThreeTxtFileHelper/UpdateBlogPaths.cs. Reads .tumblr / .tmblrpriv metadata + // files from a root\Index folder and populates Blogs.TTFolderPath in TL.db. + public static class UpdateBlogPathsRunner + { + public static int Run(string rootPath) + { + if (string.IsNullOrWhiteSpace(rootPath)) + { + Console.WriteLine("UpdateBlogPaths: rootPath is required."); + return 1; + } + + DataAccess.EnsureTTFileHelperColumnsExist(); + + string indexPath = Path.Combine(rootPath, "Index"); + if (!Directory.Exists(indexPath)) + { + Console.WriteLine($"Index folder not found at: {indexPath}"); + return 1; + } + + Console.WriteLine($"Scanning Index folder: {indexPath}"); + + var blogFiles = Directory.GetFiles(indexPath, "*.tumblr") + .Concat(Directory.GetFiles(indexPath, "*.tmblrpriv")) + .ToList(); + + Console.WriteLine($"Found {blogFiles.Count} blog metadata files"); + + int updatedCount = 0; + + foreach (var blogFile in blogFiles) + { + try + { + string blogName = Path.GetFileNameWithoutExtension(blogFile); + string jsonContent = File.ReadAllText(blogFile); + using JsonDocument doc = JsonDocument.Parse(jsonContent); + JsonElement root = doc.RootElement; + + if (root.TryGetProperty("FileDownloadLocation", out JsonElement locationElement)) + { + string? fileDownloadLocation = locationElement.GetString(); + if (!string.IsNullOrWhiteSpace(fileDownloadLocation)) + { + DataAccess.SetBlogTTFolderPath(blogName, fileDownloadLocation); + updatedCount++; + Console.WriteLine($"Updated {blogName}: {fileDownloadLocation}"); + } + } + else + { + Console.WriteLine($"No FileDownloadLocation found in {blogFile}"); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error processing {blogFile}: {ex.Message}"); + } + } + + Console.WriteLine($"\nUpdated {updatedCount} blogs with TTFolderPath"); + return 0; + } + } +} diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index f3bccea..06cf030 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -12,7 +12,12 @@ "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218", "EnableFileLogging": false, "LogTraversalRecordImports": false, - "LikesRefreshCooldownDays": 7 + "LikesRefreshCooldownDays": 7, + "PathTTRoot": "", + "PathTTBackup": "", + "PathPrefixes": "prefixes.txt", + "PathCorrectionReport": "correction_report.txt", + "PathCorrectionApplied": "correction_applied.txt" }, "TumblrApi": { "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3", diff --git a/URLNotesGrabberCORE/prefixes.txt b/URLNotesGrabberCORE/prefixes.txt new file mode 100644 index 0000000..a3c45b0 --- /dev/null +++ b/URLNotesGrabberCORE/prefixes.txt @@ -0,0 +1,20 @@ +Post ID +reblog URL +Date +Has Image +Post URL +Slug +Reblog Key +Reblog Name +Summary +Quote +Body +Tags +Link +Photo URL +Photo Caption +Downloaded Files +Audio Caption +Question +Answer +Title