From 21a55250941c7e6c70b3cc279690a32a23a80574 Mon Sep 17 00:00:00 2001 From: jim Date: Thu, 16 Jul 2026 10:23:16 -0500 Subject: [PATCH] Capture multi-line Body/Downloaded files in default .txt ingest mode TraverseDirectory only ever read the single line immediately after "Body:"/"Downloaded files:", silently dropping every continuation line (multi-paragraph HTML bodies, multiple downloaded filenames). Switch to an indexed line scan so those two fields collect lines until the next recognized field prefix, matching how IngestMode.cs already handles multi-line values. --- URLNotesGrabberCORE/Program.cs | 43 +++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 3367d41..99343b0 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -1352,6 +1352,25 @@ if (shouldInsert) } + // Field prefixes TraverseDirectory recognizes as the start of a new record field. + // Used to know where a multi-line Body/Downloaded files value ends. + private static readonly string[] TraverseDirectoryFieldPrefixes = new[] + { + "Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:", + "Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:", + "Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:", + "Tags:", "Title:" + }; + + private static bool IsTraverseDirectoryFieldLine(string line) + { + foreach (var prefix in TraverseDirectoryFieldPrefixes) + { + if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + static void TraverseDirectory(string path, string outPath, List contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false) { DateTime directoryStart = DateTime.Now; @@ -1388,8 +1407,10 @@ if (shouldInsert) var urls = new List(); var reblog = new ReblogRecord(); - foreach (string line in File.ReadLines(file)) + string[] fileLines = File.ReadAllLines(file); + for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++) { + string line = fileLines[lineIndex]; if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) { if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") @@ -1441,7 +1462,15 @@ if (shouldInsert) } if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) { - reblog.downloadedFiles = line.Substring(17).Trim(); + var valueLines = new List { line.Substring(17).Trim() }; + int nextLineIndex = lineIndex + 1; + while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex])) + { + valueLines.Add(fileLines[nextLineIndex]); + nextLineIndex++; + } + reblog.downloadedFiles = string.Join("\n", valueLines).Trim(); + lineIndex = nextLineIndex - 1; } if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase)) { @@ -1453,7 +1482,15 @@ if (shouldInsert) } if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase)) { - reblog.body = line.Substring(6).Trim(); + var valueLines = new List { line.Substring(6).Trim() }; + int nextLineIndex = lineIndex + 1; + while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex])) + { + valueLines.Add(fileLines[nextLineIndex]); + nextLineIndex++; + } + reblog.body = string.Join("\n", valueLines).Trim(); + lineIndex = nextLineIndex - 1; } if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase)) {