using System.Text.RegularExpressions; using Microsoft.Extensions.Configuration; namespace URLNotesGrabberCORE { // Port of ThreeTxtFileHelper RunIngestMode. Scans a root folder for .txt files, // parses Tumblr-export fields (multi-line aware, prefix-driven), upserts each // post into TL.db.Posts via DataAccess.UpsertPostFromTextFile. public static class IngestMode { public static int Run(IConfiguration config, string[] args) { string? targetBlog = args.Length > 0 ? args[0]?.Trim() : null; if (string.IsNullOrWhiteSpace(targetBlog)) targetBlog = null; string? rootPath = config["appSettings:PathTTRoot"]; if (string.IsNullOrWhiteSpace(rootPath)) rootPath = config["appSettings:PathInput"]; if (string.IsNullOrWhiteSpace(rootPath)) { Console.WriteLine("Ingest: no root path configured. Set appSettings:PathTTRoot or appSettings:PathInput."); return 1; } if (!Directory.Exists(rootPath)) { Console.WriteLine($"Directory not found: {rootPath}"); return 1; } DataAccess.EnsureTTFileHelperColumnsExist(); string prefixesPath = config["appSettings:PathPrefixes"] ?? "prefixes.txt"; if (!Path.IsPathRooted(prefixesPath)) prefixesPath = Path.Combine(AppContext.BaseDirectory, prefixesPath); if (!File.Exists(prefixesPath)) { Console.WriteLine($"Prefixes file not found: {prefixesPath}"); return 1; } var allowedPrefixes = new HashSet(File.ReadLines(prefixesPath), StringComparer.OrdinalIgnoreCase); Console.WriteLine($"Loaded {allowedPrefixes.Count} prefixes from {prefixesPath}"); Console.WriteLine($"========== Ingest Settings =========="); Console.WriteLine($"Root path: {rootPath}"); Console.WriteLine($"Blog filter: {(targetBlog == null ? "(all blogs)" : targetBlog)}"); Console.WriteLine($"====================================="); var txtFiles = new List(); try { var dirs = Directory.GetDirectories(rootPath, "*", SearchOption.AllDirectories); Console.WriteLine($"Found {dirs.Length} directories under root."); foreach (var dir in dirs) { try { txtFiles.AddRange(Directory.GetFiles(dir, "*.txt")); } catch (Exception ex) { Console.WriteLine($" Skipping {dir}: {ex.Message}"); } } txtFiles.AddRange(Directory.GetFiles(rootPath, "*.txt")); } catch (Exception ex) { Console.WriteLine($"Error scanning root: {ex.Message}"); return 1; } Console.WriteLine($"Processing {txtFiles.Count} .txt file(s)..."); int filesProcessed = 0; int filesSkipped = 0; int postsTouched = 0; try { DataAccess.EnableImportModePragmas(); DataAccess.BeginImportSession(); foreach (string file in txtFiles) { try { string rawBlogName = Path.GetFileName(Path.GetDirectoryName(file) ?? "unknown"); string blogName = Regex.Replace(rawBlogName, @"_\d+$", ""); string postType = Path.GetFileNameWithoutExtension(file); if (targetBlog != null && !string.Equals(blogName, targetBlog, StringComparison.OrdinalIgnoreCase)) { filesSkipped++; continue; } filesProcessed++; if (filesProcessed % 50 == 0 || filesProcessed == 1) Console.WriteLine($"[{filesProcessed}/{txtFiles.Count}] {Path.GetFileName(file)}"); string currentPostId = ""; var currentPostData = new Dictionary(StringComparer.OrdinalIgnoreCase); void Flush() { if (!string.IsNullOrWhiteSpace(currentPostId) && currentPostData.Count > 0) { UpsertPostFromParsedData(blogName, currentPostId, postType, currentPostData); postsTouched++; } } var lines = File.ReadAllLines(file); int lineIndex = 0; while (lineIndex < lines.Length) { string line = lines[lineIndex]; string searchText = line.Length > 25 ? line.Substring(0, 25) : line; int colonIndex = searchText.IndexOf(": "); if (colonIndex > 0) { string prefix = line.Substring(0, colonIndex).Trim(); if (!string.IsNullOrWhiteSpace(prefix) && allowedPrefixes.Contains(prefix)) { if (string.Equals(prefix, "Post ID", StringComparison.OrdinalIgnoreCase)) { Flush(); currentPostId = line.Substring(colonIndex + 2).Trim(); currentPostData.Clear(); lineIndex++; continue; } var valueLines = new List { line.Substring(colonIndex + 2).Trim() }; int nextLineIndex = lineIndex + 1; while (nextLineIndex < lines.Length) { string nextLine = lines[nextLineIndex]; string nextSearch = nextLine.Length > 25 ? nextLine.Substring(0, 25) : nextLine; int nextColon = nextSearch.IndexOf(": "); if (nextColon > 0) { string nextPrefix = nextLine.Substring(0, nextColon).Trim(); if (!string.IsNullOrWhiteSpace(nextPrefix) && allowedPrefixes.Contains(nextPrefix)) break; } valueLines.Add(nextLine); nextLineIndex++; } currentPostData[prefix] = string.Join("\n", valueLines); lineIndex = nextLineIndex; continue; } } lineIndex++; } Flush(); } catch (Exception ex) { Console.WriteLine($" ERROR processing file {file}: {ex.Message}"); } } } finally { DataAccess.EndImportSession(); DataAccess.RestoreImportModePragmas(); } Console.WriteLine($"\nIngest complete. Files processed: {filesProcessed}. Files skipped (blog filter): {filesSkipped}. Posts touched: {postsTouched}."); return 0; } private static void UpsertPostFromParsedData(string blogName, string postId, string postType, Dictionary data) { string? G(string key) => data.TryGetValue(key, out var v) ? v : null; string? hasImageStr = G("Has Image"); bool hasImage = !string.IsNullOrWhiteSpace(hasImageStr) && (hasImageStr.Equals("true", StringComparison.OrdinalIgnoreCase) || hasImageStr == "1" || hasImageStr.Equals("yes", StringComparison.OrdinalIgnoreCase)); DataAccess.UpsertPostFromTextFile( blogName: blogName, postID: postId, reblogURL: G("reblog URL"), postDate: G("Date"), postURL: G("Post URL"), slug: G("Slug"), reblogKey: G("Reblog Key"), reblogName: G("Reblog Name"), summary: G("Summary"), quote: G("Quote"), body: G("Body"), tags: G("Tags"), link: G("Link"), photoURL: G("Photo URL"), photoCaption: G("Photo Caption"), downloadedFiles: G("Downloaded Files"), audioCaption: G("Audio Caption"), question: G("Question"), answer: G("Answer"), title: G("Title"), postType: postType, hasImage: hasImage); } } }