Capture multi-line Body/Downloaded files in default .txt ingest mode

TraverseDirectory only ever read the single line immediately after
"Body:"/"Downloaded files:", silently dropping every continuation
line (multi-paragraph HTML bodies, multiple downloaded filenames).
Switch to an indexed line scan so those two fields collect lines
until the next recognized field prefix, matching how IngestMode.cs
already handles multi-line values.
This commit is contained in:
jim
2026-07-16 10:23:16 -05:00
parent 33839930e8
commit 21a5525094
+40 -3
View File
@@ -1352,6 +1352,25 @@ if (shouldInsert)
} }
// Field prefixes TraverseDirectory recognizes as the start of a new record field.
// Used to know where a multi-line Body/Downloaded files value ends.
private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
{
"Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
"Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
"Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
"Tags:", "Title:"
};
private static bool IsTraverseDirectoryFieldLine(string line)
{
foreach (var prefix in TraverseDirectoryFieldPrefixes)
{
if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
}
return false;
}
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false) static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
{ {
DateTime directoryStart = DateTime.Now; DateTime directoryStart = DateTime.Now;
@@ -1388,8 +1407,10 @@ if (shouldInsert)
var urls = new List<string>(); var urls = new List<string>();
var reblog = new ReblogRecord(); var reblog = new ReblogRecord();
foreach (string line in File.ReadLines(file)) string[] fileLines = File.ReadAllLines(file);
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
{ {
string line = fileLines[lineIndex];
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
{ {
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
@@ -1441,7 +1462,15 @@ if (shouldInsert)
} }
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.downloadedFiles = line.Substring(17).Trim(); var valueLines = new List<string> { line.Substring(17).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
} }
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
{ {
@@ -1453,7 +1482,15 @@ if (shouldInsert)
} }
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.body = line.Substring(6).Trim(); var valueLines = new List<string> { line.Substring(6).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.body = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
} }
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
{ {