Capture multi-line Body/Downloaded files in default .txt ingest mode
TraverseDirectory only ever read the single line immediately after "Body:"/"Downloaded files:", silently dropping every continuation line (multi-paragraph HTML bodies, multiple downloaded filenames). Switch to an indexed line scan so those two fields collect lines until the next recognized field prefix, matching how IngestMode.cs already handles multi-line values.
This commit is contained in:
@@ -1352,6 +1352,25 @@ if (shouldInsert)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Field prefixes TraverseDirectory recognizes as the start of a new record field.
|
||||||
|
// Used to know where a multi-line Body/Downloaded files value ends.
|
||||||
|
private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
|
||||||
|
{
|
||||||
|
"Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
|
||||||
|
"Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
|
||||||
|
"Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
|
||||||
|
"Tags:", "Title:"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static bool IsTraverseDirectoryFieldLine(string line)
|
||||||
|
{
|
||||||
|
foreach (var prefix in TraverseDirectoryFieldPrefixes)
|
||||||
|
{
|
||||||
|
if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
|
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
|
||||||
{
|
{
|
||||||
DateTime directoryStart = DateTime.Now;
|
DateTime directoryStart = DateTime.Now;
|
||||||
@@ -1388,8 +1407,10 @@ if (shouldInsert)
|
|||||||
var urls = new List<string>();
|
var urls = new List<string>();
|
||||||
var reblog = new ReblogRecord();
|
var reblog = new ReblogRecord();
|
||||||
|
|
||||||
foreach (string line in File.ReadLines(file))
|
string[] fileLines = File.ReadAllLines(file);
|
||||||
|
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
|
||||||
{
|
{
|
||||||
|
string line = fileLines[lineIndex];
|
||||||
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
||||||
@@ -1441,7 +1462,15 @@ if (shouldInsert)
|
|||||||
}
|
}
|
||||||
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
reblog.downloadedFiles = line.Substring(17).Trim();
|
var valueLines = new List<string> { line.Substring(17).Trim() };
|
||||||
|
int nextLineIndex = lineIndex + 1;
|
||||||
|
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
|
||||||
|
{
|
||||||
|
valueLines.Add(fileLines[nextLineIndex]);
|
||||||
|
nextLineIndex++;
|
||||||
|
}
|
||||||
|
reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
|
||||||
|
lineIndex = nextLineIndex - 1;
|
||||||
}
|
}
|
||||||
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
@@ -1453,7 +1482,15 @@ if (shouldInsert)
|
|||||||
}
|
}
|
||||||
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
reblog.body = line.Substring(6).Trim();
|
var valueLines = new List<string> { line.Substring(6).Trim() };
|
||||||
|
int nextLineIndex = lineIndex + 1;
|
||||||
|
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
|
||||||
|
{
|
||||||
|
valueLines.Add(fileLines[nextLineIndex]);
|
||||||
|
nextLineIndex++;
|
||||||
|
}
|
||||||
|
reblog.body = string.Join("\n", valueLines).Trim();
|
||||||
|
lineIndex = nextLineIndex - 1;
|
||||||
}
|
}
|
||||||
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user