Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0ff80a0fd3 | ||
|
|
4df73367fb | ||
|
|
a437fa87d3 | ||
|
|
32a1583efd | ||
|
|
03676432bd |
@@ -22,18 +22,18 @@ This document provides essential context for AI agents working with URLNotesGrab
|
||||
```powershell
|
||||
dotnet build
|
||||
dotnet run # Process all files in input directory
|
||||
dotnet run -- -parse [blogname] # Process specific blog
|
||||
dotnet run -- -test [blogname] [postID] # Test API for specific post
|
||||
dotnet run -- --parse [blogname] # Process specific blog
|
||||
dotnet run -- --test [blogname] [postID] # Test API for specific post
|
||||
```
|
||||
|
||||
### Command-Line Interface
|
||||
- `-parse [blogname]`: Parse text files for specific blog
|
||||
- `-test [blogname] [postID]`: Test API note collection
|
||||
- `-posts`: Export post blogs to file
|
||||
- `-blogs`: Export blog list to file
|
||||
- `-collect`: Collect notes for all posts in DB
|
||||
- `-blogsR`: Export reply blogs to file
|
||||
- `-blogsO [start] [stop]`: Export blogs within range
|
||||
- `--parse [blogname]`: Parse text files for specific blog
|
||||
- `--test [blogname] [postID]`: Test API note collection
|
||||
- `--posts`: Export post blogs to file
|
||||
- `--blogs`: Export blog list to file
|
||||
- `--collect`: Collect notes for all posts in DB
|
||||
- `--blogsR`: Export reply blogs to file
|
||||
- `--blogsO [start] [stop]`: Export blogs within range
|
||||
|
||||
## Project Conventions
|
||||
|
||||
|
||||
@@ -518,8 +518,12 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
if (ownsConnection) connection.Open();
|
||||
|
||||
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
|
||||
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
||||
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
|
||||
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
||||
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
||||
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||
updateCommand.Parameters.AddWithValue("@blogName", blogName);
|
||||
updateCommand.Parameters.AddWithValue("@postID", postID);
|
||||
|
||||
updateCommand.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
+145
-91
@@ -13,12 +13,27 @@ namespace URLNotesGrabberCORE
|
||||
internal class Program
|
||||
{
|
||||
|
||||
static void Main(string[] args)
|
||||
static int Main(string[] args)
|
||||
{
|
||||
// Reset console color on exit (including Ctrl+C)
|
||||
Console.CancelKeyPress += (s, e) => Console.ResetColor();
|
||||
AppDomain.CurrentDomain.ProcessExit += (s, e) => Console.ResetColor();
|
||||
|
||||
try
|
||||
{
|
||||
return Run(args);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Turn configuration errors / unguarded indexers into a quiet, deterministic exit code.
|
||||
Console.Error.WriteLine(ex.Message);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
static int Run(string[] args)
|
||||
{
|
||||
int exitCode = 0;
|
||||
IConfiguration config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
|
||||
@@ -33,31 +48,34 @@ namespace URLNotesGrabberCORE
|
||||
List<string> filteredArgs = new List<string>();
|
||||
for (int i = 0; i < args.Length; i++)
|
||||
{
|
||||
if (string.Equals(args[i], "-force", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
|
||||
if (args[i] == "--")
|
||||
{
|
||||
// POSIX end-of-options: everything after is a literal operand.
|
||||
for (int j = i + 1; j < args.Length; j++) filteredArgs.Add(args[j]);
|
||||
break;
|
||||
}
|
||||
|
||||
if (string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
forceIgnoreCooldown = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(args[i], "-api3", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
|
||||
if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
apiSectionName = "TumblrApi3";
|
||||
apiExplicitlySet = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(args[i], "-api4", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase))
|
||||
if (string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
apiSectionName = "TumblrApi4";
|
||||
apiExplicitlySet = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(args[i], "-api", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(args[i], "--api", StringComparison.OrdinalIgnoreCase))
|
||||
if (string.Equals(args[i], "--api", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
||||
{
|
||||
@@ -67,13 +85,12 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("--Missing API section after -api/--api. Using default TumblrApi.--");
|
||||
Console.WriteLine("--Missing API section after --api. Using default TumblrApi.--");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(args[i], "-start", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
|
||||
if (string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
||||
{
|
||||
@@ -82,7 +99,7 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("--Missing blog name after -start/--start. Ignoring.--");
|
||||
Console.WriteLine("--Missing blog name after --start. Ignoring.--");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -147,57 +164,23 @@ namespace URLNotesGrabberCORE
|
||||
switch (args[0])
|
||||
{
|
||||
case "-?":
|
||||
Console.WriteLine("\t Parse .txt files to find blogs");
|
||||
|
||||
Console.WriteLine("-?\t Usage help");
|
||||
|
||||
Console.WriteLine("-parse\t Parse .txt files with specified blogname");
|
||||
|
||||
Console.WriteLine("-test\t Calls API for given blogname and postID");
|
||||
|
||||
Console.WriteLine("-posts\t For each Post in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
|
||||
|
||||
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
||||
|
||||
Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
|
||||
|
||||
Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
|
||||
|
||||
Console.WriteLine("-likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
|
||||
|
||||
Console.WriteLine("-force\t (with -likes) Ignore cooldown and refresh every fully-backfilled blog");
|
||||
|
||||
Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
|
||||
|
||||
Console.WriteLine("-api3\t Use TumblrApi3 settings from appsettings.json");
|
||||
|
||||
Console.WriteLine("-api4\t Use TumblrApi4 settings from appsettings.json");
|
||||
|
||||
Console.WriteLine("-start [blogname]\t Start traversal alphabetically at this blog name");
|
||||
|
||||
Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
|
||||
|
||||
Console.WriteLine("-ingest [blogname]\t Ingest Tumblr .txt exports from appSettings:PathTTRoot into TL.db (all blogs, or single blog if name given)");
|
||||
|
||||
Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
|
||||
|
||||
Console.WriteLine("-revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
|
||||
|
||||
Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
||||
|
||||
Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
||||
|
||||
Console.WriteLine("-updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from <root>\\Index and set Blogs.TTFolderPath");
|
||||
|
||||
Console.WriteLine("-importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");
|
||||
|
||||
case "-h":
|
||||
case "--help":
|
||||
PrintHelp();
|
||||
break;
|
||||
|
||||
case "-parse":
|
||||
|
||||
case "-V":
|
||||
case "--version":
|
||||
Console.WriteLine(System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown");
|
||||
break;
|
||||
|
||||
case "--parse":
|
||||
if (args.Length < 2)
|
||||
{
|
||||
Console.WriteLine("Usage: --parse <blogname>");
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
string blogNameToParse = args[1];
|
||||
int postsAdded = 0;
|
||||
try
|
||||
@@ -212,23 +195,23 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine($"Total posts added: {postsAdded}");
|
||||
break;
|
||||
|
||||
case "-test":
|
||||
case "--test":
|
||||
Console.WriteLine("Test command not implemented");
|
||||
break;
|
||||
|
||||
case "-post":
|
||||
case "--post":
|
||||
TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
|
||||
break;
|
||||
|
||||
case "-posts": //write post's blogs to file
|
||||
case "--posts": //write post's blogs to file
|
||||
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
|
||||
break;
|
||||
|
||||
case "-blogs": //write blogs to file
|
||||
case "--blogs": //write blogs to file
|
||||
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
|
||||
break;
|
||||
|
||||
case "-collect": //collect notes from all posts
|
||||
case "--collect": //collect notes from all posts
|
||||
bool withoutNotesOnly = true;
|
||||
DateTime? beforeDate = DateTime.Now;
|
||||
bool explicitDateSupplied = false;
|
||||
@@ -236,6 +219,7 @@ namespace URLNotesGrabberCORE
|
||||
if (args.Length < 2)
|
||||
{
|
||||
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -268,6 +252,7 @@ namespace URLNotesGrabberCORE
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -297,14 +282,14 @@ namespace URLNotesGrabberCORE
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
case "-blogsR": //collect notes from all posts
|
||||
case "--blogsR": //collect notes from all posts
|
||||
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
|
||||
break;
|
||||
|
||||
case "-blogsO": //collect notes from all posts
|
||||
case "--blogsO": //collect notes from all posts
|
||||
int from = 1, to = 999999, top = 100;
|
||||
|
||||
if (args[1] is not null && args[2] is not null && args[3] is not null)
|
||||
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
|
||||
{
|
||||
from = int.Parse(args[1]);
|
||||
to = int.Parse(args[2]);
|
||||
@@ -313,14 +298,16 @@ namespace URLNotesGrabberCORE
|
||||
else
|
||||
{
|
||||
Console.WriteLine("--Expected FROM TO--");
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
||||
break;
|
||||
|
||||
case "-bop": //collect notes from all posts
|
||||
case "--bop": //collect notes from all posts
|
||||
from = 1; to = 999999; top = 100;
|
||||
|
||||
if (args[1] is not null && args[2] is not null && args[3] is not null)
|
||||
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
|
||||
{
|
||||
from = int.Parse(args[1]);
|
||||
to = int.Parse(args[2]);
|
||||
@@ -329,66 +316,70 @@ namespace URLNotesGrabberCORE
|
||||
else
|
||||
{
|
||||
Console.WriteLine("--Expected FROM TO--");
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
||||
break;
|
||||
|
||||
case "-replies": //update reply text
|
||||
case "--replies": //update reply text
|
||||
CollectMissingReplyText().GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
case "-likes":
|
||||
case "--likes":
|
||||
string likeBlog = args.Length > 1 ? args[1] : null;
|
||||
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
|
||||
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
case "-urldump":
|
||||
case "--urldump":
|
||||
DumpUrls(settings.GetValue<string>("PathOutputUrls"));
|
||||
break;
|
||||
|
||||
case "-ingest":
|
||||
IngestMode.Run(config, args.Skip(1).ToArray());
|
||||
case "--ingest":
|
||||
exitCode = IngestMode.Run(config, args.Skip(1).ToArray());
|
||||
break;
|
||||
|
||||
case "-output":
|
||||
OutputMode.Run(config);
|
||||
case "--output":
|
||||
exitCode = OutputMode.Run(config);
|
||||
break;
|
||||
|
||||
case "-revert":
|
||||
RevertMode.Run(config, args.Length > 1 ? args[1] : null);
|
||||
case "--revert":
|
||||
exitCode = RevertMode.Run(config, args.Length > 1 ? args[1] : null);
|
||||
break;
|
||||
|
||||
case "-correct":
|
||||
case "--correct":
|
||||
{
|
||||
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
||||
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
||||
var correctArgs = args.Skip(1)
|
||||
.Where(a => !string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) && !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase))
|
||||
.Where(a => !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase))
|
||||
.ToArray();
|
||||
CorrectMode.Run(config, correctArgs, applyChanges);
|
||||
exitCode = CorrectMode.Run(config, correctArgs, applyChanges);
|
||||
break;
|
||||
}
|
||||
|
||||
case "-updatepaths":
|
||||
case "--updatepaths":
|
||||
{
|
||||
string rootPath = args.Length > 1 ? args[1] : (settings.GetValue<string>("PathTTRoot") ?? settings.GetValue<string>("PathInput") ?? string.Empty);
|
||||
UpdateBlogPathsRunner.Run(rootPath);
|
||||
exitCode = UpdateBlogPathsRunner.Run(rootPath);
|
||||
break;
|
||||
}
|
||||
|
||||
case "-importposts":
|
||||
case "--importposts":
|
||||
{
|
||||
if (args.Length < 2)
|
||||
{
|
||||
Console.WriteLine("Usage: -importposts <path-to-legacy-posts.db>");
|
||||
Console.WriteLine("Usage: --importposts <path-to-legacy-posts.db>");
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
LegacyPostsDbImporter.Run(args[1]);
|
||||
exitCode = LegacyPostsDbImporter.Run(args[1]);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
Console.WriteLine("** Unknown Command ** " + args[0]);
|
||||
exitCode = 2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -397,6 +388,69 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
System.Console.WriteLine("<fin>:/");
|
||||
//System.Console.ReadKey();
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
static void PrintHelp()
|
||||
{
|
||||
Console.WriteLine("\t Parse .txt files to find blogs");
|
||||
|
||||
Console.WriteLine("-?, -h, --help\t Usage help");
|
||||
|
||||
Console.WriteLine("-V, --version\t Print the application version");
|
||||
|
||||
Console.WriteLine("--\t End of options: treat every following token as a literal operand");
|
||||
|
||||
Console.WriteLine("--parse\t Parse .txt files with specified blogname");
|
||||
|
||||
Console.WriteLine("--test\t Calls API for given blogname and postID");
|
||||
|
||||
Console.WriteLine("--post\t Traverse the input directory tree checking .txt files for corruption");
|
||||
|
||||
Console.WriteLine("--posts\t For each Post in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("--collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
|
||||
|
||||
Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");
|
||||
|
||||
Console.WriteLine("--blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
|
||||
|
||||
Console.WriteLine("--bop [from] [to] [top]\t Write ALL blog names to file, limited by FROM TO TOP range arguments");
|
||||
|
||||
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
|
||||
|
||||
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
|
||||
|
||||
Console.WriteLine("--force\t (with --likes) Ignore cooldown and refresh every fully-backfilled blog");
|
||||
|
||||
Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
|
||||
|
||||
Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json");
|
||||
|
||||
Console.WriteLine("--api4\t Use TumblrApi4 settings from appsettings.json");
|
||||
|
||||
Console.WriteLine("--start [blogname]\t Start traversal alphabetically at this blog name");
|
||||
|
||||
Console.WriteLine("--api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
|
||||
|
||||
Console.WriteLine("--ingest [blogname]\t Ingest Tumblr .txt exports from appSettings:PathTTRoot into TL.db (all blogs, or single blog if name given)");
|
||||
|
||||
Console.WriteLine("--output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
|
||||
|
||||
Console.WriteLine("--revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
|
||||
|
||||
Console.WriteLine("--correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
||||
|
||||
Console.WriteLine("--correct --apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
||||
|
||||
Console.WriteLine("--updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from <root>\\Index and set Blogs.TTFolderPath");
|
||||
|
||||
Console.WriteLine("--importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("Exit status: 0 = success; 1 = unexpected error; 2 = usage error (unknown command or bad/missing arguments)");
|
||||
}
|
||||
|
||||
static void WritePostBlogsToFile(string outPath)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"profiles": {
|
||||
"URLNotesGrabberCORE": {
|
||||
"commandName": "Project",
|
||||
"commandLineArgs": "-collect 1 -api4"
|
||||
"commandLineArgs": "--collect 1 --api4"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user