jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote master into local master

Browse Source
This commit is contained in:
jim
2026-07-22 12:12:33 -05:00
parent 3e2b287737 16147b273e
commit f9e1d2100b
3 changed files with 110 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+60 -9
View File
@@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration;
using System.Configuration;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Text.RegularExpressions;
namespace URLNotesGrabberCORE
@@ -140,7 +139,10 @@ namespace URLNotesGrabberCORE
Console.SetOut(dualLogger);
}
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
string? containsListSetting = settings.GetValue<string>("ContainsList");
if (string.IsNullOrEmpty(containsListSetting))
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
List<string> contains = containsListSetting.Split(',').ToList();
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
@@ -175,6 +177,12 @@ namespace URLNotesGrabberCORE
break;
case "--parse":
if (args.Length < 2)
{
Console.WriteLine("Usage: --parse <blogname>");
exitCode = 2;
break;
}
string blogNameToParse = args[1];
int postsAdded = 0;
try
@@ -283,7 +291,7 @@ namespace URLNotesGrabberCORE
case "--blogsO": //collect notes from all posts
int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
@@ -293,6 +301,7 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine("--Expected FROM TO--");
exitCode = 2;
break;
}
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
@@ -300,7 +309,7 @@ namespace URLNotesGrabberCORE
case "--bop": //collect notes from all posts
from = 1; to = 999999; top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
@@ -310,6 +319,7 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine("--Expected FROM TO--");
exitCode = 2;
break;
}
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
@@ -1412,6 +1422,25 @@ if (shouldInsert)
}
// Field prefixes TraverseDirectory recognizes as the start of a new record field.
// Used to know where a multi-line Body/Downloaded files value ends.
private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
{
"Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
"Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
"Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
"Tags:", "Title:"
};
private static bool IsTraverseDirectoryFieldLine(string line)
{
foreach (var prefix in TraverseDirectoryFieldPrefixes)
{
if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
}
return false;
}
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
{
DateTime directoryStart = DateTime.Now;
@@ -1448,8 +1477,10 @@ if (shouldInsert)
var urls = new List<string>();
var reblog = new ReblogRecord();
foreach (string line in File.ReadLines(file))
string[] fileLines = File.ReadAllLines(file);
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
{
string line = fileLines[lineIndex];
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
{
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
@@ -1470,7 +1501,7 @@ if (shouldInsert)
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, false);
reblog.title, false, rootURL: reblog.rootURL);
recordImportStopwatch.Stop();
postsAdded++;
@@ -1495,9 +1526,21 @@ if (shouldInsert)
{
reblog.reblogName = line.Substring(13).Trim();
}
if (line.StartsWith(@"Reblog root url:", StringComparison.OrdinalIgnoreCase))
{
reblog.rootURL = line.Substring(16).Trim();
}
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
{
reblog.downloadedFiles = line.Substring(17).Trim();
var valueLines = new List<string> { line.Substring(17).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
}
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
{
@@ -1509,7 +1552,15 @@ if (shouldInsert)
}
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
{
reblog.body = line.Substring(6).Trim();
var valueLines = new List<string> { line.Substring(6).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.body = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
}
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
{
@@ -1598,7 +1649,7 @@ if (shouldInsert)
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, true);
reblog.title, true, rootURL: reblog.rootURL);
recordImportStopwatch.Stop();
postsAdded++;