jimandClaude Opus 5
8f4177a0c9
TL.db syncs between machines whose absolute paths differ, so a single TTFolderPath column cannot be correct on both at once -- the stored paths are only trustworthy on the machine that wrote them. That made --updatepaths a mandatory prelude to every --output rather than the one-time setup step it looks like. --output now refreshes the column from the TumblThree Index metadata before exporting. The root comes from the first non-flag argument, else appSettings:PathTTRoot. With no root available it says so and exports whatever TL.db already holds; a root whose Index folder is missing is a hard stop, since silently exporting stale paths is the failure this change exists to prevent. --norefresh skips the refresh for a pure export. The scan is extracted from UpdateBlogPathsRunner.Run into a reusable Scan() that returns counts instead of only printing them, so --updatepaths keeps its per-file detail while --output prints a single summary line rather than a few hundred lines ahead of the export. Verified against a throwaway database: a stale cross-machine path is repaired and the export lands in the correct local folder; no configured root warns and continues (exit 0); a missing Index folder stops (exit 1); --norefresh skips the refresh and exports (exit 0). Co-Authored-By: Claude Opus 5 <[email protected]>
1741 lines
87 KiB
C#
1741 lines
87 KiB
C#
using System.Threading.RateLimiting;
|
|
using Microsoft.Extensions.Http.Resilience;
|
|
using Microsoft.Extensions;
|
|
using Microsoft.Extensions.Configuration;
|
|
using System.Configuration;
|
|
using System.Threading;
|
|
using Microsoft.Extensions.Diagnostics.Latency;
|
|
using System.Text.RegularExpressions;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
internal class Program
|
|
{
|
|
|
|
static int Main(string[] args)
|
|
{
|
|
// Reset console color on exit (including Ctrl+C)
|
|
Console.CancelKeyPress += (s, e) => Console.ResetColor();
|
|
AppDomain.CurrentDomain.ProcessExit += (s, e) => Console.ResetColor();
|
|
|
|
try
|
|
{
|
|
return Run(args);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Turn configuration errors / unguarded indexers into a quiet, deterministic exit code.
|
|
Console.Error.WriteLine(ex.Message);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static int Run(string[] args)
|
|
{
|
|
int exitCode = 0;
|
|
IConfiguration config = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
|
|
.AddCommandLine(args)
|
|
.Build();
|
|
var settings = config.GetSection("appSettings");
|
|
|
|
string apiSectionName = "TumblrApi";
|
|
bool apiExplicitlySet = false;
|
|
string startFromBlogName = string.Empty;
|
|
bool forceIgnoreCooldown = false;
|
|
List<string> filteredArgs = new List<string>();
|
|
for (int i = 0; i < args.Length; i++)
|
|
{
|
|
if (args[i] == "--")
|
|
{
|
|
// POSIX end-of-options: everything after is a literal operand.
|
|
for (int j = i + 1; j < args.Length; j++) filteredArgs.Add(args[j]);
|
|
break;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
forceIgnoreCooldown = true;
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
apiSectionName = "TumblrApi3";
|
|
apiExplicitlySet = true;
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
apiSectionName = "TumblrApi4";
|
|
apiExplicitlySet = true;
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--api", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
|
{
|
|
apiSectionName = args[i + 1].Trim();
|
|
apiExplicitlySet = true;
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Missing API section after --api. Using default TumblrApi.--");
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
|
{
|
|
startFromBlogName = args[i + 1].Trim();
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Missing blog name after --start. Ignoring.--");
|
|
}
|
|
continue;
|
|
}
|
|
|
|
filteredArgs.Add(args[i]);
|
|
}
|
|
|
|
args = filteredArgs.ToArray();
|
|
|
|
string? dbPath = config["appSettings:PathDB"];
|
|
if (string.IsNullOrEmpty(dbPath))
|
|
throw new InvalidOperationException("PathDB is not configured in appsettings.json");
|
|
|
|
ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null);
|
|
|
|
// Setup Dual Logging
|
|
bool enableFileLogging = settings.GetValue("EnableFileLogging", true);
|
|
if (enableFileLogging)
|
|
{
|
|
string logPath = "console_output.log";
|
|
if (File.Exists(logPath))
|
|
{
|
|
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
|
|
string archiveDirectory = "logs";
|
|
|
|
// Ensure the archive directory exists
|
|
if (!Directory.Exists(archiveDirectory))
|
|
{
|
|
Directory.CreateDirectory(archiveDirectory);
|
|
}
|
|
|
|
string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
|
|
File.Move(logPath, newPath);
|
|
}
|
|
|
|
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
|
|
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
|
|
Console.SetOut(dualLogger);
|
|
}
|
|
|
|
string? containsListSetting = settings.GetValue<string>("ContainsList");
|
|
if (string.IsNullOrEmpty(containsListSetting))
|
|
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
|
|
List<string> contains = containsListSetting.Split(',').ToList();
|
|
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
|
|
|
|
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
|
{
|
|
int postsAdded = 0;
|
|
try
|
|
{
|
|
DataAccess.EnableImportModePragmas();
|
|
DataAccess.BeginImportSession();
|
|
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports);
|
|
}
|
|
finally
|
|
{
|
|
DataAccess.EndImportSession();
|
|
DataAccess.RestoreImportModePragmas();
|
|
}
|
|
Console.WriteLine($"Total posts added: {postsAdded}");
|
|
}
|
|
else
|
|
{
|
|
switch (args[0])
|
|
{
|
|
case "-?":
|
|
case "-h":
|
|
case "--help":
|
|
PrintHelp();
|
|
break;
|
|
|
|
case "-V":
|
|
case "--version":
|
|
Console.WriteLine(System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown");
|
|
break;
|
|
|
|
case "--parse":
|
|
if (args.Length < 2)
|
|
{
|
|
Console.WriteLine("Usage: --parse <blogname>");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
string blogNameToParse = args[1];
|
|
int postsAdded = 0;
|
|
try
|
|
{
|
|
DataAccess.EnableImportModePragmas();
|
|
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse, startFromBlogName, logTraversalRecordImports);
|
|
}
|
|
finally
|
|
{
|
|
DataAccess.RestoreImportModePragmas();
|
|
}
|
|
Console.WriteLine($"Total posts added: {postsAdded}");
|
|
break;
|
|
|
|
case "--test":
|
|
Console.WriteLine("Test command not implemented");
|
|
break;
|
|
|
|
case "--post":
|
|
TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
|
|
break;
|
|
|
|
case "--posts": //write post's blogs to file
|
|
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
|
|
break;
|
|
|
|
case "--blogs": //write blogs to file
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
|
|
break;
|
|
|
|
case "--collect": //collect notes from all posts
|
|
bool withoutNotesOnly = true;
|
|
DateTime? beforeDate = DateTime.Now;
|
|
bool explicitDateSupplied = false;
|
|
|
|
if (args.Length < 2)
|
|
{
|
|
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
|
|
// Parse withoutNotesOnly flag
|
|
if (args.Length > 1 && args[1] is not null)
|
|
{
|
|
if (args[1] == "1")
|
|
{
|
|
withoutNotesOnly = true;
|
|
Console.WriteLine("Parsed");
|
|
}
|
|
else
|
|
{
|
|
withoutNotesOnly = false;
|
|
Console.WriteLine("--NOT Parsed");
|
|
}
|
|
|
|
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
|
|
}
|
|
|
|
// Parse optional beforeDate parameter
|
|
if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
|
|
{
|
|
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
|
{
|
|
beforeDate = parsedDate;
|
|
explicitDateSupplied = true;
|
|
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
|
|
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
|
|
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
|
|
bool managedCollectRun = false;
|
|
if (!withoutNotesOnly && !explicitDateSupplied)
|
|
{
|
|
DataAccess.EnsureCollectRunStateTableExists();
|
|
var runState = DataAccess.GetCollectRunState();
|
|
if (runState != null && !runState.Value.complete)
|
|
{
|
|
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
|
|
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
|
|
}
|
|
else
|
|
{
|
|
beforeDate = DateTime.Now;
|
|
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
|
|
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
|
|
}
|
|
managedCollectRun = true;
|
|
}
|
|
|
|
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
|
|
break;
|
|
|
|
case "--blogsR": //collect notes from all posts
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
|
|
break;
|
|
|
|
case "--blogsO": //collect notes from all posts
|
|
int from = 1, to = 999999, top = 100;
|
|
|
|
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
|
|
{
|
|
from = int.Parse(args[1]);
|
|
to = int.Parse(args[2]);
|
|
top = int.Parse(args[3]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Expected FROM TO--");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
|
break;
|
|
|
|
case "--bop": //collect notes from all posts
|
|
from = 1; to = 999999; top = 100;
|
|
|
|
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
|
|
{
|
|
from = int.Parse(args[1]);
|
|
to = int.Parse(args[2]);
|
|
top = int.Parse(args[3]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Expected FROM TO--");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
|
break;
|
|
|
|
case "--replies": //update reply text
|
|
CollectMissingReplyText().GetAwaiter().GetResult();
|
|
break;
|
|
|
|
case "--likes":
|
|
string likeBlog = args.Length > 1 ? args[1] : null;
|
|
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
|
|
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
|
|
break;
|
|
|
|
case "--urldump":
|
|
DumpUrls(settings.GetValue<string>("PathOutputUrls"));
|
|
break;
|
|
|
|
case "--ingest":
|
|
exitCode = IngestMode.Run(config, args.Skip(1).ToArray());
|
|
break;
|
|
|
|
case "--output":
|
|
exitCode = OutputMode.Run(config, args.Skip(1).ToArray());
|
|
break;
|
|
|
|
case "--revert":
|
|
exitCode = RevertMode.Run(config, args.Length > 1 ? args[1] : null);
|
|
break;
|
|
|
|
case "--correct":
|
|
{
|
|
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
|
var correctArgs = args.Skip(1)
|
|
.Where(a => !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase))
|
|
.ToArray();
|
|
exitCode = CorrectMode.Run(config, correctArgs, applyChanges);
|
|
break;
|
|
}
|
|
|
|
case "--updatepaths":
|
|
{
|
|
string rootPath = args.Length > 1 ? args[1] : (settings.GetValue<string>("PathTTRoot") ?? settings.GetValue<string>("PathInput") ?? string.Empty);
|
|
exitCode = UpdateBlogPathsRunner.Run(rootPath);
|
|
break;
|
|
}
|
|
|
|
case "--importposts":
|
|
{
|
|
if (args.Length < 2)
|
|
{
|
|
Console.WriteLine("Usage: --importposts <path-to-legacy-posts.db>");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
exitCode = LegacyPostsDbImporter.Run(args[1]);
|
|
break;
|
|
}
|
|
|
|
default:
|
|
Console.WriteLine("** Unknown Command ** " + args[0]);
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
System.Console.WriteLine("<fin>:/");
|
|
//System.Console.ReadKey();
|
|
return exitCode;
|
|
}
|
|
|
|
static void PrintHelp()
|
|
{
|
|
Console.WriteLine("\t Parse .txt files to find blogs");
|
|
|
|
Console.WriteLine("-?, -h, --help\t Usage help");
|
|
|
|
Console.WriteLine("-V, --version\t Print the application version");
|
|
|
|
Console.WriteLine("--\t End of options: treat every following token as a literal operand");
|
|
|
|
Console.WriteLine("--parse\t Parse .txt files with specified blogname");
|
|
|
|
Console.WriteLine("--test\t Calls API for given blogname and postID");
|
|
|
|
Console.WriteLine("--post\t Traverse the input directory tree checking .txt files for corruption");
|
|
|
|
Console.WriteLine("--posts\t For each Post in DB, write blogname to file");
|
|
|
|
Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");
|
|
|
|
Console.WriteLine("--collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
|
|
|
|
Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");
|
|
|
|
Console.WriteLine("--blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
|
|
|
|
Console.WriteLine("--bop [from] [to] [top]\t Write ALL blog names to file, limited by FROM TO TOP range arguments");
|
|
|
|
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
|
|
|
|
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
|
|
|
|
Console.WriteLine("--force\t (with --likes) Ignore cooldown and refresh every fully-backfilled blog");
|
|
|
|
Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
|
|
|
|
Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json");
|
|
|
|
Console.WriteLine("--api4\t Use TumblrApi4 settings from appsettings.json");
|
|
|
|
Console.WriteLine("--start [blogname]\t Start traversal alphabetically at this blog name");
|
|
|
|
Console.WriteLine("--api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
|
|
|
|
Console.WriteLine("--ingest [blogname]\t Ingest Tumblr .txt exports from appSettings:PathTTRoot into TL.db (all blogs, or single blog if name given)");
|
|
|
|
Console.WriteLine("--output [rootPath]\t Refresh Blogs.TTFolderPath from <root>\\Index (or appSettings:PathTTRoot), then export posts from TL.db back to .txt files in each blog's folder");
|
|
|
|
Console.WriteLine("--output --norefresh\t Export without refreshing TTFolderPath first");
|
|
|
|
Console.WriteLine("--revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
|
|
|
|
Console.WriteLine("--correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
|
|
|
Console.WriteLine("--correct --apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
|
|
|
Console.WriteLine("--updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from <root>\\Index and set Blogs.TTFolderPath");
|
|
|
|
Console.WriteLine("--importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine("Exit status: 0 = success; 1 = unexpected error; 2 = usage error (unknown command or bad/missing arguments); 3 = incomplete (--collect paused on a rate limit, or skipped posts after transient API failures) - relaunch to resume");
|
|
}
|
|
|
|
static void WritePostBlogsToFile(string outPath)
|
|
{
|
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
List<string> blogs = new List<string>();
|
|
foreach (var post in posts)
|
|
{
|
|
blogs.Add(post.Item1);
|
|
}
|
|
blogs = blogs.Distinct().ToList();
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
|
|
{
|
|
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
DataAccess.UpdateBlogOutput(blog);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
|
|
{
|
|
List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
DataAccess.UpdateBlogOutput(blog);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
|
|
{
|
|
List<Tuple<string, long>> posts = DataAccess.GetReplies();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var post in posts)
|
|
{
|
|
Console.WriteLine(@"https://{0}.tumblr.com/post/{1}", post.Item1, post.Item2);
|
|
sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void DumpUrls(string? outPath)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(outPath))
|
|
{
|
|
Console.WriteLine("--PathOutputUrls is not configured in appsettings.json--");
|
|
return;
|
|
}
|
|
|
|
Console.WriteLine($"Starting URL extraction to {outPath}...");
|
|
HashSet<string> uniqueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
Regex urlRegex = new Regex(@"https?://[^\s""'<>]+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
|
|
|
int rowsProcessed = 0;
|
|
foreach (var texts in DataAccess.GetAllPostTextColumns())
|
|
{
|
|
rowsProcessed++;
|
|
if (rowsProcessed % 10000 == 0)
|
|
{
|
|
Console.WriteLine($"Scanned {rowsProcessed} rows... found {uniqueUrls.Count} unique URLs so far.");
|
|
}
|
|
|
|
foreach (var text in texts)
|
|
{
|
|
var matches = urlRegex.Matches(text);
|
|
foreach (Match match in matches)
|
|
{
|
|
uniqueUrls.Add(match.Value);
|
|
}
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"Scan complete. Sorting and saving {uniqueUrls.Count} distinct URLs...");
|
|
var sortedUrls = uniqueUrls.ToList();
|
|
sortedUrls.Sort();
|
|
|
|
File.WriteAllLines(outPath, sortedUrls);
|
|
Console.WriteLine($"Saved URLs to {outPath}");
|
|
}
|
|
|
|
protected static bool ContainsAny(string input, List<string> contains)
|
|
{
|
|
if (string.IsNullOrEmpty(input)) return false;
|
|
|
|
|
|
foreach (string item in contains)
|
|
{
|
|
if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
protected static string NormalizeBlogFolderName(string folderName)
|
|
{
|
|
// Archive tools suffix duplicate blog folders with _1, _2, ... _10 and beyond. Strip only a
|
|
// trailing numeric suffix: unanchored substring removal ate the "_1" inside "_10" and left the
|
|
// "0" welded to the name (zomb-eh_10 -> zomb-eh0), and mangled any blog whose real name
|
|
// contains "_1". A blog name is never a prefix of itself plus "_<digits>", so this is safe.
|
|
return System.Text.RegularExpressions.Regex.Replace(folderName, @"_\d+$", "");
|
|
}
|
|
|
|
static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
|
|
{
|
|
const int MaxPages = 10;
|
|
try
|
|
{
|
|
Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}.tumblr.com/post/{postID}");
|
|
|
|
int replyCount = 0;
|
|
int rowsUpdated = 0;
|
|
int emptyReplyCount = 0;
|
|
long pageTimestamp = 0;
|
|
int page = 0;
|
|
bool sawHealthyResponse = false;
|
|
|
|
while (page < MaxPages)
|
|
{
|
|
page++;
|
|
await Task.Delay(2000);
|
|
var key = ApiKeyPool.GetCurrentKey();
|
|
var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);
|
|
|
|
if (postsResponse?.statusCode == "TooManyRequests")
|
|
{
|
|
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
|
|
ApiKeyPool.MarkRateLimited(key, retry);
|
|
Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
|
|
return;
|
|
}
|
|
|
|
if (postsResponse?.meta?.status != 429)
|
|
ApiKeyPool.MarkAvailable(key);
|
|
|
|
if (postsResponse?.meta?.status == 200)
|
|
sawHealthyResponse = true;
|
|
|
|
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
|
|
{
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"[Reply Text] No notes returned for {blogName}.tumblr.com/post/{postID} on page {page} (before_timestamp={pageTimestamp})");
|
|
Console.ForegroundColor = prevColor;
|
|
Console.WriteLine($"[Reply Text] meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
|
|
var raw = postsResponse?.rawJson ?? string.Empty;
|
|
if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
|
|
Console.WriteLine($"[Reply Text] raw: {raw}");
|
|
|
|
bool isNotFound = postsResponse?.meta?.status == 404
|
|
|| string.Equals(postsResponse?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
|
|
if (isNotFound)
|
|
{
|
|
var notFoundColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"[Reply Text] Post {blogName}.tumblr.com/post/{postID} returned 404 - marking NotFound=1 and replies '?'");
|
|
Console.ForegroundColor = notFoundColor;
|
|
DataAccess.UpdatePostMarkNotFound(blogName, postID);
|
|
rowsUpdated += DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
|
|
emptyReplyCount += rowsUpdated;
|
|
}
|
|
break;
|
|
}
|
|
|
|
long lastNoteTimestamp = 0;
|
|
foreach (var note in postsResponse.response.notes)
|
|
{
|
|
if (note.timestamp > 0)
|
|
lastNoteTimestamp = note.timestamp;
|
|
|
|
if (note.type == "reply")
|
|
{
|
|
if (!string.IsNullOrEmpty(note.reply_text))
|
|
{
|
|
rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
|
|
replyCount++;
|
|
|
|
string displayText = note.reply_text.Length > 100
|
|
? note.reply_text.Substring(0, 100) + "..."
|
|
: note.reply_text;
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine($" [{note.blog_name}] {displayText}");
|
|
Console.ForegroundColor = previousColor;
|
|
}
|
|
else
|
|
{
|
|
rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
|
|
emptyReplyCount++;
|
|
Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
|
|
}
|
|
}
|
|
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
|
|
{
|
|
rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
|
|
replyCount++;
|
|
|
|
string displayText = note.reply_text.Length > 100
|
|
? note.reply_text.Substring(0, 100) + "..."
|
|
: note.reply_text;
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Cyan;
|
|
Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)");
|
|
Console.ForegroundColor = previousColor;
|
|
}
|
|
}
|
|
|
|
if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
|
|
{
|
|
break;
|
|
}
|
|
pageTimestamp = lastNoteTimestamp;
|
|
}
|
|
|
|
// If pagination finished without updating any reply rows but the API responded healthily
|
|
// at least once, mark the post's outstanding '.' replies '?' so the work queue releases it.
|
|
if (rowsUpdated == 0 && sawHealthyResponse)
|
|
{
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"[Reply Text] Marking outstanding replies for {blogName}.tumblr.com/post/{postID} '?' (healthy 200 OK, no matching reply notes after {page} page(s))");
|
|
Console.ForegroundColor = prevColor;
|
|
rowsUpdated = DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
|
|
emptyReplyCount += rowsUpdated;
|
|
}
|
|
|
|
Console.WriteLine($"[Reply Text] Done {blogName}.tumblr.com/post/{postID}: notes={replyCount}, rowsUpdated={rowsUpdated}, empty='?'={emptyReplyCount}, pages={page}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}.tumblr.com/post/{postID}: {ex.Message}");
|
|
Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
|
|
}
|
|
}
|
|
|
|
static async Task CollectMissingReplyText()
|
|
{
|
|
try
|
|
{
|
|
// Ensure the replyText column exists
|
|
DataAccess.EnsureReplyTextColumnExists();
|
|
|
|
Console.WriteLine("Starting collection of missing reply text...");
|
|
Console.WriteLine("Processing replies directly from Tumblr API.");
|
|
Console.WriteLine();
|
|
|
|
int totalProcessedCount = 0;
|
|
int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
|
|
|
Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
|
|
Console.WriteLine();
|
|
|
|
while (true)
|
|
{
|
|
// Re-query each iteration so posts whose replies got filled in as a side-effect
|
|
// of a previous post's update are skipped without burning an API call.
|
|
var batch = DataAccess.GetRepliesWithFilledText(limit: 1);
|
|
|
|
if (batch is null || batch.Count == 0)
|
|
{
|
|
Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
|
|
break;
|
|
}
|
|
|
|
var reply = batch[0];
|
|
var blogName = reply.Item1;
|
|
var postID = reply.Item2;
|
|
var timestamp = reply.Item3;
|
|
|
|
await FetchAndStoreReplyText(blogName, postID, timestamp);
|
|
totalProcessedCount++;
|
|
|
|
int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
|
double completionPct = initialTotal > 0
|
|
? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
|
|
: 100.0;
|
|
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Cyan;
|
|
Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
|
|
Console.ForegroundColor = previousColor;
|
|
|
|
if (totalProcessedCount % 50 == 0)
|
|
{
|
|
await Task.Delay(2000);
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Reply Text] Error: {ex.Message}");
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
}
|
|
|
|
static ConsoleColor GetMatchedCounterColor(long matchedCount)
|
|
{
|
|
if (matchedCount <= 0)
|
|
return ConsoleColor.White;
|
|
|
|
return ((matchedCount - 1) % 3) switch
|
|
{
|
|
0 => ConsoleColor.Red,
|
|
1 => ConsoleColor.Green,
|
|
_ => ConsoleColor.Blue
|
|
};
|
|
}
|
|
|
|
static void WriteLikesTotalsLine(string blogName, string label, long parsedForBlog, long matchedForBlog, int likedCountForBlog)
|
|
{
|
|
if (likedCountForBlog > 0)
|
|
{
|
|
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
|
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
|
|
|
Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: ");
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
|
|
Console.Write(matchedForBlog);
|
|
Console.ForegroundColor = previousColor;
|
|
Console.WriteLine($"/{likedCountForBlog} ({matchedPct:F2}%)");
|
|
}
|
|
else
|
|
{
|
|
Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog} | Matched: ");
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
|
|
Console.Write(matchedForBlog);
|
|
Console.ForegroundColor = previousColor;
|
|
Console.WriteLine();
|
|
}
|
|
}
|
|
|
|
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
|
|
{
|
|
try
|
|
{
|
|
DataAccess.EnsureBlogsLikesColumnsExist();
|
|
|
|
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
|
|
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
|
|
|
|
if (blogsToProcess.Count == 0)
|
|
{
|
|
Console.WriteLine("No blogs found to process likes for.");
|
|
return;
|
|
}
|
|
|
|
int backfillCount = blogsToProcess.Count(b => b.Item2 == 0);
|
|
int refreshCount = blogsToProcess.Count(b => b.Item2 == 1);
|
|
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh).");
|
|
|
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
|
{
|
|
// 1/sec average, matching --collect: the CDN reacts to aggregate traffic from the IP,
|
|
// not to per-command rates.
|
|
PermitLimit = 60,
|
|
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
|
QueueLimit = 1,
|
|
Window = TimeSpan.FromMinutes(1),
|
|
SegmentsPerWindow = 60,
|
|
AutoReplenishment = true
|
|
});
|
|
|
|
for (int i = 0; i < blogsToProcess.Count; i++)
|
|
{
|
|
var blogInfo = blogsToProcess[i];
|
|
int remaining = blogsToProcess.Count - i - 1;
|
|
string blogName = blogInfo.Item1;
|
|
int likesPulled = blogInfo.Item2;
|
|
long cursor = blogInfo.Item3;
|
|
long storedNewestTs = blogInfo.Item4;
|
|
|
|
bool isRefresh = likesPulled == 1;
|
|
long parsedForBlog = 0;
|
|
long matchedForBlog = 0;
|
|
int likedCountForBlog = 0;
|
|
long observedMaxLikedTs = storedNewestTs;
|
|
int newInsertedInRefresh = 0;
|
|
|
|
// Refresh always starts from the top (newest) and walks backward until it crosses
|
|
// the stored high-water mark. Backfill resumes from its last persisted cursor.
|
|
if (isRefresh) cursor = 0;
|
|
|
|
string mode = isRefresh ? "REFRESH" : "BACKFILL";
|
|
Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
|
|
|
|
bool hasMoreLikes = true;
|
|
bool isFirstPage = true;
|
|
|
|
while (hasMoreLikes)
|
|
{
|
|
// Wait for a permit rather than giving up on one: the limiter paces the loop, it is
|
|
// not a failure condition. Only one acquire is ever pending, so QueueLimit = 1 suffices.
|
|
using RateLimitLease lease = await limiter.AcquireAsync(1);
|
|
if (!lease.IsAcquired)
|
|
{
|
|
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
|
return;
|
|
}
|
|
|
|
var key = ApiKeyPool.GetCurrentKey();
|
|
var response = await APIAccess.GrabLikes(key, blogName, cursor);
|
|
|
|
if (response?.statusCode == "TooManyRequests")
|
|
{
|
|
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
|
|
ApiKeyPool.MarkRateLimited(key, retry);
|
|
|
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
|
continue;
|
|
}
|
|
|
|
if (response.meta?.status != 429)
|
|
ApiKeyPool.MarkAvailable(key);
|
|
|
|
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
|
|
if (isRefresh)
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
else
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
|
|
break;
|
|
}
|
|
|
|
if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
|
|
{
|
|
Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
|
|
if (isRefresh)
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
else
|
|
{
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
|
|
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
|
|
}
|
|
hasMoreLikes = false;
|
|
break;
|
|
}
|
|
|
|
if (response.response.liked_count > 0)
|
|
likedCountForBlog = response.response.liked_count;
|
|
|
|
Console.WriteLine($"[Likes] [Fetched {response.response.liked_posts.Count}] {blogName}");
|
|
|
|
bool crossedHighWaterMark = false;
|
|
|
|
foreach (var post in response.response.liked_posts)
|
|
{
|
|
parsedForBlog++;
|
|
|
|
long postID = 0;
|
|
try { postID = Convert.ToInt64(post.id); } catch { continue; }
|
|
|
|
// liked_timestamp is when the user liked the post (matches the `before` cursor semantics).
|
|
// It's the only reliable field for the refresh stop condition.
|
|
long likedTs = 0;
|
|
try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { }
|
|
|
|
if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs)
|
|
{
|
|
Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh.");
|
|
crossedHighWaterMark = true;
|
|
break;
|
|
}
|
|
|
|
if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs;
|
|
|
|
string authorBlog = post.blog_name?.ToString() ?? ".";
|
|
string postURL = post.post_url?.ToString() ?? ".";
|
|
string date = post.date?.ToString() ?? ".";
|
|
|
|
// Legacy format fields
|
|
string body = post.body?.ToString() ?? ".";
|
|
string summary = post.summary?.ToString() ?? ".";
|
|
string slug = post.slug?.ToString() ?? ".";
|
|
string reblogURL = post.source_url?.ToString() ?? ".";
|
|
string reblogName = post.reblogged_from_name?.ToString() ?? post.source_title?.ToString() ?? ".";
|
|
string rootBlogName = post.reblogged_root_name?.ToString() ?? ".";
|
|
string rootURL = post.reblogged_root_url?.ToString() ?? ".";
|
|
|
|
// Quick check for tags
|
|
string tags = ".";
|
|
if (post.tags != null)
|
|
{
|
|
try { tags = string.Join(", ", post.tags); }
|
|
catch { }
|
|
}
|
|
|
|
bool hasImage = false;
|
|
string photoURL = ".";
|
|
string photoCaption = ".";
|
|
|
|
if (post.photos != null)
|
|
{
|
|
hasImage = true;
|
|
try
|
|
{
|
|
var firstPhoto = post.photos[0];
|
|
if (firstPhoto != null)
|
|
{
|
|
if (firstPhoto.original_size != null)
|
|
photoURL = firstPhoto.original_size.url?.ToString() ?? ".";
|
|
photoCaption = firstPhoto.caption?.ToString() ?? ".";
|
|
}
|
|
} catch { }
|
|
}
|
|
else if (body.IndexOf("<img", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
hasImage = true;
|
|
}
|
|
|
|
// Optional fields
|
|
string quote = ".";
|
|
string audioCaption = ".";
|
|
string question = ".";
|
|
string answer = ".";
|
|
string title = post.title?.ToString() ?? ".";
|
|
string downloadedFiles = ".";
|
|
string reblogKey = post.reblog_key?.ToString() ?? ".";
|
|
string link = ".";
|
|
|
|
// Only insert if any of the data contains strings from ContainsList
|
|
bool shouldInsert = false;
|
|
string matchedFieldName = string.Empty;
|
|
|
|
// Let's check a few fields that usually have URLs or relevant info that might match ContainsList
|
|
var fieldsToCheck = new (string Name, string Value)[] {
|
|
("postURL", postURL),
|
|
("authorBlog", authorBlog),
|
|
("reblogURL", reblogURL),
|
|
("reblogName", reblogName),
|
|
("rootBlogName", rootBlogName),
|
|
("rootURL", rootURL),
|
|
("summary", summary),
|
|
("body", body),
|
|
("tags", tags),
|
|
("photoURL", photoURL),
|
|
("photoCaption", photoCaption),
|
|
("quote", quote),
|
|
("question", question),
|
|
("answer", answer),
|
|
("downloadedFiles", downloadedFiles),
|
|
("slug", slug)
|
|
};
|
|
|
|
foreach (var field in fieldsToCheck)
|
|
{
|
|
if (ContainsAny(field.Value, contains))
|
|
{
|
|
shouldInsert = true;
|
|
matchedFieldName = field.Name;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (shouldInsert)
|
|
{
|
|
matchedForBlog++;
|
|
if (isRefresh) newInsertedInRefresh++;
|
|
Console.WriteLine($"[Likes] [Matched] {blogName} | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
|
|
await Task.Delay(3000);
|
|
DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
|
|
reblogName, summary, quote, body, tags, link, photoURL,
|
|
photoCaption, downloadedFiles, audioCaption, question, answer,
|
|
title, hasImage, true, rootBlogName: rootBlogName, rootURL: rootURL);
|
|
}
|
|
}
|
|
|
|
WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
|
|
|
// Branch B: refresh terminates as soon as we crossed the high-water mark.
|
|
if (isRefresh && crossedHighWaterMark)
|
|
{
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
hasMoreLikes = false;
|
|
break;
|
|
}
|
|
|
|
// Branch A: on the very first page, capture & persist the newest liked_timestamp
|
|
// so subsequent refresh runs (after backfill completes) have a stopping point.
|
|
if (!isRefresh && isFirstPage && observedMaxLikedTs > 0)
|
|
{
|
|
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
|
|
}
|
|
isFirstPage = false;
|
|
|
|
// Determine the next BeforeCursor.
|
|
long nextCursor = 0;
|
|
if (response.response._links?.next?.query_params != null)
|
|
{
|
|
long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
|
|
}
|
|
|
|
if (!isRefresh)
|
|
{
|
|
// Persist cursor progress after every page so resume is always up-to-date
|
|
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
|
|
}
|
|
|
|
if (nextCursor > 0)
|
|
{
|
|
cursor = nextCursor;
|
|
Console.WriteLine($"[Likes] Pagination Next Cursor: {cursor}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
|
|
if (isRefresh)
|
|
{
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
}
|
|
else
|
|
{
|
|
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist);
|
|
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
|
|
}
|
|
hasMoreLikes = false;
|
|
}
|
|
|
|
Console.WriteLine($"[{remaining} remaining] Done with {blogName}");
|
|
await Task.Delay(1000); // 1-second delay between pages
|
|
}
|
|
|
|
if (isRefresh)
|
|
Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}");
|
|
WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
|
}
|
|
|
|
Console.WriteLine("[Likes] [Likes collection complete]");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error collecting likes: {ex.Message}");
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
}
|
|
|
|
// Backoff between in-place retries of a transient infrastructure failure. Most CDN 403s and edge
|
|
// 5xxs clear within a few seconds, so retrying here saves the post its single attempt for the pass.
|
|
static readonly int[] TransientBackoffSeconds = { 1, 4, 10 };
|
|
|
|
// Fetches one page, retrying transient failures in place. Rate limits are returned to the caller
|
|
// untouched — those are handled by pausing the whole run, not by retrying this post.
|
|
static async Task<Root> FetchNotesPage(Tuple<string, long, long, long> post, string beforeTimestamp)
|
|
{
|
|
Root response = null!;
|
|
|
|
for (int attempt = 0; ; attempt++)
|
|
{
|
|
var key = ApiKeyPool.GetCurrentKey();
|
|
response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
|
|
|
|
if (response.statusCode == "TooManyRequests")
|
|
{
|
|
ApiKeyPool.MarkRateLimited(key, response.retryInSeconds > 0 ? response.retryInSeconds : 60);
|
|
return response;
|
|
}
|
|
|
|
if (!response.transientFailure)
|
|
{
|
|
// Only a response that actually reached the API says anything about the key's standing.
|
|
ApiKeyPool.MarkAvailable(key);
|
|
return response;
|
|
}
|
|
|
|
if (attempt >= TransientBackoffSeconds.Length)
|
|
return response;
|
|
|
|
int delay = TransientBackoffSeconds[attempt];
|
|
Console.WriteLine($"[Transient] retry {attempt + 1}/{TransientBackoffSeconds.Length} in {delay}s");
|
|
await Task.Delay(delay * 1000);
|
|
}
|
|
}
|
|
|
|
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
|
|
{
|
|
try
|
|
{
|
|
static bool IsNotFound(Root? r)
|
|
{
|
|
return (r?.meta != null && r.meta.status == 404) || string.Equals(r?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
int APICount = DataAccess.GetAPICount();
|
|
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
|
|
|
|
var allNotes = new List<dynamic>();
|
|
int page = 1;
|
|
string beforeTimestamp = post.Item3.ToString();
|
|
bool hasReplies = false;
|
|
const int maxPages = 500;
|
|
var response = await FetchNotesPage(post, beforeTimestamp);
|
|
|
|
if (response.statusCode == "TooManyRequests")
|
|
return "TooManyRequests";
|
|
|
|
if (response.transientFailure)
|
|
{
|
|
Console.WriteLine($"[Skip] {post.Item1}/{post.Item2} — {response.statusCode} after {TransientBackoffSeconds.Length} retries");
|
|
return "Transient";
|
|
}
|
|
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2}");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
Thread.Sleep(1000);
|
|
return "NotFound";
|
|
}
|
|
|
|
// Pagination loop
|
|
while (true)
|
|
{
|
|
int noteCount = response?.response?.notes?.Count ?? 0;
|
|
Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");
|
|
|
|
if (response?.response == null)
|
|
{
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (empty response payload)");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return "NotFound";
|
|
}
|
|
Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
|
|
return "FAILURE";
|
|
}
|
|
if (response.response.notes == null)
|
|
{
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (notes payload missing)");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return "NotFound";
|
|
}
|
|
Console.WriteLine("##### Notes Null - WHY? ###");
|
|
return "FAILURE";
|
|
}
|
|
|
|
// Accumulate notes
|
|
allNotes.AddRange(response.response.notes);
|
|
|
|
// Pagination: check for next
|
|
var nextLink = response.response._links?.next;
|
|
if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
|
|
{
|
|
Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
|
|
break;
|
|
}
|
|
|
|
beforeTimestamp = nextLink.query_params.before_timestamp;
|
|
page++;
|
|
if (page > maxPages)
|
|
{
|
|
Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}.tumblr.com/post/{post.Item2}. Aborting further pagination.");
|
|
break;
|
|
}
|
|
|
|
response = await FetchNotesPage(post, beforeTimestamp);
|
|
if (response.statusCode == "TooManyRequests")
|
|
return "TooManyRequests";
|
|
|
|
if (response.transientFailure)
|
|
{
|
|
Console.WriteLine($"[Skip] {post.Item1}/{post.Item2} — {response.statusCode} on page {page} after {TransientBackoffSeconds.Length} retries");
|
|
return "Transient";
|
|
}
|
|
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}.tumblr.com/post/{post.Item2}");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return "NotFound";
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
|
|
|
|
// Process all accumulated notes
|
|
foreach (var note in allNotes)
|
|
{
|
|
note.reblog_parent_blog_name = post.Item1;
|
|
note.post_id = post.Item2.ToString();
|
|
if (note.type == "reply")
|
|
hasReplies = true;
|
|
DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
|
|
}
|
|
|
|
return "Success";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
return "UNKNOWN";
|
|
}
|
|
|
|
// Consecutive transient failures that mean the API edge is rejecting traffic wholesale rather than
|
|
// blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
|
|
const int MaxConsecutiveTransient = 10;
|
|
|
|
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
|
|
{
|
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
|
|
|
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
|
|
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
|
|
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
|
|
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
|
|
|
|
int skipped = 0;
|
|
int consecutiveTransient = 0;
|
|
|
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
|
{
|
|
// 1/sec average. Sustained higher rates draw CDN-level 403s that the API's own rate-limit
|
|
// headers never warn about, so this sits well under the per-key quota on purpose.
|
|
PermitLimit = 60,
|
|
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
|
QueueLimit = 1,
|
|
Window = TimeSpan.FromMinutes(1),
|
|
SegmentsPerWindow = 60,
|
|
AutoReplenishment = true
|
|
});
|
|
|
|
try
|
|
{
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
while (posts.Count > 0)
|
|
{
|
|
// First post not yet attempted this process. If all remaining have been attempted, the pass
|
|
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
|
|
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
|
|
if (post == null)
|
|
{
|
|
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
|
|
break;
|
|
}
|
|
|
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
|
|
|
// Wait for a permit rather than giving up on one: the limiter paces the loop, it is
|
|
// not a failure condition. Only one acquire is ever pending, so QueueLimit = 1 suffices.
|
|
using RateLimitLease lease = await limiter.AcquireAsync(1);
|
|
if (!lease.IsAcquired)
|
|
{
|
|
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
|
return 3; // abort without completing the run so a later launch resumes
|
|
}
|
|
|
|
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
|
|
string status = await GrabNotes(post);
|
|
|
|
if (status == "Transient")
|
|
{
|
|
// Retries in GrabNotes are already exhausted. Skip the post so the pass can make
|
|
// progress; it stays unmarked in the DB, so the next launch picks it up again.
|
|
attempted.Add((post.Item1, post.Item2));
|
|
skipped++;
|
|
consecutiveTransient++;
|
|
|
|
if (consecutiveTransient >= MaxConsecutiveTransient)
|
|
{
|
|
Console.WriteLine($"[Abort] {consecutiveTransient} consecutive transient failures - the API edge is rejecting traffic. Pausing run; relaunch to resume. ({skipped} post(s) skipped)");
|
|
return 3;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
consecutiveTransient = 0;
|
|
|
|
if (status == "Success")
|
|
{
|
|
attempted.Add((post.Item1, post.Item2));
|
|
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
|
}
|
|
else if (status == "NotFound")
|
|
{
|
|
attempted.Add((post.Item1, post.Item2));
|
|
Console.WriteLine("GrabNotes Result: NotFound");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
}
|
|
else if (status == "TooManyRequests")
|
|
{
|
|
// Throttle, not a real per-post failure: don't consume this post's single attempt.
|
|
// Abort the pass without completing so a later launch resumes against the same cutoff.
|
|
Console.WriteLine($"GrabNotes Result: TooManyRequests - pausing run; relaunch to resume. ({skipped} post(s) skipped)");
|
|
return 3;
|
|
}
|
|
else
|
|
{
|
|
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
|
|
attempted.Add((post.Item1, post.Item2));
|
|
Console.WriteLine("GrabNotes Result: " + status);
|
|
}
|
|
}
|
|
|
|
// Re-fetch the updated list after processing the current post
|
|
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
|
}
|
|
}
|
|
|
|
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
|
|
if (managedRun)
|
|
{
|
|
DataAccess.CompleteCollectRun();
|
|
Console.WriteLine("Full re-check run complete.");
|
|
}
|
|
|
|
if (skipped > 0)
|
|
{
|
|
Console.WriteLine($"Pass finished with {skipped} post(s) skipped after transient failures; relaunch to retry them.");
|
|
return 3;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
|
|
// Field prefixes TraverseDirectory recognizes as the start of a new record field.
|
|
// Used to know where a multi-line Body/Downloaded files value ends.
|
|
private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
|
|
{
|
|
"Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
|
|
"Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
|
|
"Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
|
|
"Tags:", "Title:"
|
|
};
|
|
|
|
private static bool IsTraverseDirectoryFieldLine(string line)
|
|
{
|
|
foreach (var prefix in TraverseDirectoryFieldPrefixes)
|
|
{
|
|
if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
|
|
{
|
|
DateTime directoryStart = DateTime.Now;
|
|
int directoryRecordsImported = 0;
|
|
Console.WriteLine($"[Directory Start] {path} | {directoryStart:yyyy-MM-dd HH:mm:ss.fff}");
|
|
|
|
try
|
|
{
|
|
// Get all directories in the current directory and sort them alphabetically
|
|
var directories = Directory.GetDirectories(path);
|
|
Array.Sort(directories, StringComparer.InvariantCulture);
|
|
|
|
foreach (var directory in directories)
|
|
{
|
|
Console.WriteLine("Directory: " + directory);
|
|
TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName, startFromBlogName, logRecordImports); // Recursively traverse subdirectories
|
|
}
|
|
|
|
try
|
|
{
|
|
bool headerWasWritten = false;
|
|
// Process all files in the current directory
|
|
foreach (var file in Directory.GetFiles(path))
|
|
{
|
|
string normalizedDirectoryName = NormalizeBlogFolderName(new DirectoryInfo(path).Name);
|
|
bool isAtOrAfterStart = string.IsNullOrWhiteSpace(startFromBlogName) || string.Compare(normalizedDirectoryName, startFromBlogName, StringComparison.OrdinalIgnoreCase) >= 0;
|
|
|
|
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)
|
|
&& (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)
|
|
&& isAtOrAfterStart)
|
|
{
|
|
try
|
|
{
|
|
var urls = new List<string>();
|
|
var reblog = new ReblogRecord();
|
|
|
|
string[] fileLines = File.ReadAllLines(file);
|
|
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
|
|
{
|
|
string line = fileLines[lineIndex];
|
|
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
|
{
|
|
if (!reblog.reblogURL.Contains("deactivated")
|
|
&& reblog.reblogURL.Length != 0
|
|
&& ContainsAny(reblog.reblogURL, contains))
|
|
{
|
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
if (!headerWasWritten)
|
|
{
|
|
headerWasWritten = true;
|
|
}
|
|
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
|
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
|
|
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
reblog.title, false, rootURL: reblog.rootURL);
|
|
recordImportStopwatch.Stop();
|
|
|
|
postsAdded++;
|
|
directoryRecordsImported++;
|
|
if (logRecordImports)
|
|
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
|
|
|
// Output hyperlink and post date
|
|
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
|
}
|
|
}
|
|
reblog = new ReblogRecord();
|
|
reblog.postID = line.Substring(9).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
//reblog = new ReblogRecord();
|
|
|
|
reblog.reblogURL = line.Substring(12).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.reblogName = line.Substring(13).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog root url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.rootURL = line.Substring(16).Trim();
|
|
}
|
|
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var valueLines = new List<string> { line.Substring(17).Trim() };
|
|
int nextLineIndex = lineIndex + 1;
|
|
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
|
|
{
|
|
valueLines.Add(fileLines[nextLineIndex]);
|
|
nextLineIndex++;
|
|
}
|
|
reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
|
|
lineIndex = nextLineIndex - 1;
|
|
}
|
|
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.reblogKey = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.date = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var valueLines = new List<string> { line.Substring(6).Trim() };
|
|
int nextLineIndex = lineIndex + 1;
|
|
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
|
|
{
|
|
valueLines.Add(fileLines[nextLineIndex]);
|
|
nextLineIndex++;
|
|
}
|
|
reblog.body = string.Join("\n", valueLines).Trim();
|
|
lineIndex = nextLineIndex - 1;
|
|
}
|
|
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.postURL = line.Substring(10).Trim();
|
|
}
|
|
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.answer = line.Substring(8).Trim();
|
|
}
|
|
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.audioCaption = line.Substring(15).Trim();
|
|
}
|
|
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.blogName = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.link = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.photoCaption = line.Substring(15).Trim();
|
|
}
|
|
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.photoURL = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.question = line.Substring(10).Trim();
|
|
}
|
|
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.quote = line.Substring(7).Trim();
|
|
}
|
|
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.slug = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.summary = line.Substring(9).Trim();
|
|
}
|
|
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.tags = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.title = line.Substring(7).Trim();
|
|
}
|
|
|
|
if ((reblog.downloadedFiles != "."
|
|
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
|
{
|
|
if ((ContainsAny(reblog.downloadedFiles, contains)
|
|
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
|
//&& !reblog.reblogURL.Contains("deactivated")
|
|
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
|
{
|
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
if (!headerWasWritten)
|
|
{
|
|
headerWasWritten = true;
|
|
}
|
|
//if (reblog.reblogURL.Contains("/blog/private")
|
|
// || reblog.body.Contains("/blog/private"))
|
|
//{
|
|
// Console.WriteLine(reblog.postID);
|
|
// Console.WriteLine(reblog.postURL);
|
|
// Console.WriteLine(reblog.date);
|
|
// Console.WriteLine(reblog.body);
|
|
// Console.WriteLine(reblog.reblogKey);
|
|
// Console.WriteLine(reblog.reblogURL);
|
|
// Console.WriteLine(reblog.reblogName);
|
|
// Console.WriteLine(reblog.downloadedFiles);
|
|
//}
|
|
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
|
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
|
|
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
reblog.title, true, rootURL: reblog.rootURL);
|
|
recordImportStopwatch.Stop();
|
|
|
|
postsAdded++;
|
|
directoryRecordsImported++;
|
|
if (logRecordImports)
|
|
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
|
|
|
// Output hyperlink and post date
|
|
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
|
}
|
|
}
|
|
}
|
|
urls.Sort();
|
|
|
|
|
|
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
//{
|
|
// foreach (string line in urls.Distinct())
|
|
// {
|
|
// sw.WriteLine(line);
|
|
// }
|
|
//}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine("The file could not be read:");
|
|
Console.WriteLine(e.Message);
|
|
}
|
|
}
|
|
// Add your file processing logic here
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"An error occurred: {ex.Message}");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
DateTime directoryEnd = DateTime.Now;
|
|
TimeSpan elapsed = directoryEnd - directoryStart;
|
|
Console.WriteLine($"[Directory End] {path} | {directoryEnd:yyyy-MM-dd HH:mm:ss.fff} | Duration: {elapsed:hh\\:mm\\:ss\\.fff} | Records Imported: {directoryRecordsImported}");
|
|
}
|
|
}
|
|
|
|
|
|
static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
|
|
{
|
|
var directories = Directory.GetDirectories(path);
|
|
Array.Sort(directories, StringComparer.InvariantCulture);
|
|
|
|
foreach (var directory in directories)
|
|
{
|
|
//Console.WriteLine("Directory: " + directory);
|
|
TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
|
|
}
|
|
try
|
|
{
|
|
bool headerWasWritten = false;
|
|
foreach (var file in Directory.GetFiles(path))
|
|
{
|
|
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
|
|
{
|
|
try
|
|
{
|
|
var urls = new List<string>();
|
|
var reblog = new ReblogRecord();
|
|
|
|
foreach (string line in File.ReadLines(file))
|
|
{
|
|
if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
|
|
{
|
|
Console.WriteLine(file);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine("The file could not be read:");
|
|
Console.WriteLine(e.Message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"An error occurred: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
}
|