jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture multi-line Body/Downloaded files in default .txt ingest mode

Browse Source 
TraverseDirectory only ever read the single line immediately after
"Body:"/"Downloaded files:", silently dropping every continuation
line (multi-paragraph HTML bodies, multiple downloaded filenames).
Switch to an indexed line scan so those two fields collect lines
until the next recognized field prefix, matching how IngestMode.cs
already handles multi-line values.
This commit is contained in:
jim
2026-07-16 10:23:16 -05:00
parent 33839930e8
commit 21a5525094
1 changed files with 40 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+40 -3
View File
@@ -1352,6 +1352,25 @@ if (shouldInsert)
}
// Field prefixes TraverseDirectory recognizes as the start of a new record field.
// Used to know where a multi-line Body/Downloaded files value ends.
private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
{
"Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
"Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
"Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
"Tags:", "Title:"
};
private static bool IsTraverseDirectoryFieldLine(string line)
{
foreach (var prefix in TraverseDirectoryFieldPrefixes)
{
if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
}
return false;
}
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
{
DateTime directoryStart = DateTime.Now;
@@ -1388,8 +1407,10 @@ if (shouldInsert)
var urls = new List<string>();
var reblog = new ReblogRecord();
foreach (string line in File.ReadLines(file))
string[] fileLines = File.ReadAllLines(file);
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
{
string line = fileLines[lineIndex];
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
{
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
@@ -1441,7 +1462,15 @@ if (shouldInsert)
}
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
{
reblog.downloadedFiles = line.Substring(17).Trim();
var valueLines = new List<string> { line.Substring(17).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
}
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
{
@@ -1453,7 +1482,15 @@ if (shouldInsert)
}
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
{
reblog.body = line.Substring(6).Trim();
var valueLines = new List<string> { line.Substring(6).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.body = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
}
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
{