jimandClaude Opus 4.8
003a504d5e
A non-JSON response body (CDN 403/5xx HTML, empty body, transport error) never reached the Tumblr API, so it says nothing about the post being fetched. These were recorded as FAILURE, which consumed the post's single attempt for the pass and cleared the API key's rate-limit flag on the way through. Classify them as Root.transientFailure and retry in place (1s/4s/10s) before skipping. Skipped posts stay unmarked in the DB so a later launch retries them. Ten consecutive transient failures now aborts the pass rather than skipping post-by-post against an edge refusing all traffic. Also: - MarkAvailable() only on a response that reached the API, and it is now a no-op when the key was not flagged (was writing to the DB and logging on every single call) - Only a real 429 counts as a rate limit; stop inferring one from X-RateLimit-* headers, which Tumblr sends on every response - Limiters pace with AcquireAsync instead of AttemptAcquire, which did not wait and aborted the run once a window was saturated - Throttle --collect and --likes from 300/min to 60/min - Log one line per transient failure instead of the HTML body and stack trace; keep full detail only for a 2xx that fails to parse - --collect returns exit 3 when a pass ends incomplete Co-Authored-By: Claude Opus 4.8 <[email protected]>
1693 lines
83 KiB
C#
1693 lines
83 KiB
C#
using System.Threading.RateLimiting;
|
|
using Microsoft.Extensions.Http.Resilience;
|
|
using Microsoft.Extensions;
|
|
using Microsoft.Extensions.Configuration;
|
|
using System.Configuration;
|
|
using System.Threading;
|
|
using Microsoft.Extensions.Diagnostics.Latency;
|
|
using static System.Runtime.InteropServices.JavaScript.JSType;
|
|
using System.Text.RegularExpressions;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
internal class Program
|
|
{
|
|
|
|
static int Main(string[] args)
|
|
{
|
|
// Reset console color on exit (including Ctrl+C)
|
|
Console.CancelKeyPress += (s, e) => Console.ResetColor();
|
|
AppDomain.CurrentDomain.ProcessExit += (s, e) => Console.ResetColor();
|
|
|
|
try
|
|
{
|
|
return Run(args);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Turn configuration errors / unguarded indexers into a quiet, deterministic exit code.
|
|
Console.Error.WriteLine(ex.Message);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static int Run(string[] args)
|
|
{
|
|
int exitCode = 0;
|
|
IConfiguration config = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
|
|
.AddCommandLine(args)
|
|
.Build();
|
|
var settings = config.GetSection("appSettings");
|
|
|
|
string apiSectionName = "TumblrApi";
|
|
bool apiExplicitlySet = false;
|
|
string startFromBlogName = string.Empty;
|
|
bool forceIgnoreCooldown = false;
|
|
List<string> filteredArgs = new List<string>();
|
|
for (int i = 0; i < args.Length; i++)
|
|
{
|
|
if (args[i] == "--")
|
|
{
|
|
// POSIX end-of-options: everything after is a literal operand.
|
|
for (int j = i + 1; j < args.Length; j++) filteredArgs.Add(args[j]);
|
|
break;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
forceIgnoreCooldown = true;
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
apiSectionName = "TumblrApi3";
|
|
apiExplicitlySet = true;
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
apiSectionName = "TumblrApi4";
|
|
apiExplicitlySet = true;
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--api", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
|
{
|
|
apiSectionName = args[i + 1].Trim();
|
|
apiExplicitlySet = true;
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Missing API section after --api. Using default TumblrApi.--");
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
|
{
|
|
startFromBlogName = args[i + 1].Trim();
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Missing blog name after --start. Ignoring.--");
|
|
}
|
|
continue;
|
|
}
|
|
|
|
filteredArgs.Add(args[i]);
|
|
}
|
|
|
|
args = filteredArgs.ToArray();
|
|
|
|
string? dbPath = config["appSettings:PathDB"];
|
|
if (string.IsNullOrEmpty(dbPath))
|
|
throw new InvalidOperationException("PathDB is not configured in appsettings.json");
|
|
|
|
ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null);
|
|
|
|
// Setup Dual Logging
|
|
bool enableFileLogging = settings.GetValue("EnableFileLogging", true);
|
|
if (enableFileLogging)
|
|
{
|
|
string logPath = "console_output.log";
|
|
if (File.Exists(logPath))
|
|
{
|
|
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
|
|
string archiveDirectory = "logs";
|
|
|
|
// Ensure the archive directory exists
|
|
if (!Directory.Exists(archiveDirectory))
|
|
{
|
|
Directory.CreateDirectory(archiveDirectory);
|
|
}
|
|
|
|
string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
|
|
File.Move(logPath, newPath);
|
|
}
|
|
|
|
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
|
|
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
|
|
Console.SetOut(dualLogger);
|
|
}
|
|
|
|
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
|
|
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
|
|
|
|
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
|
{
|
|
int postsAdded = 0;
|
|
try
|
|
{
|
|
DataAccess.EnableImportModePragmas();
|
|
DataAccess.BeginImportSession();
|
|
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports);
|
|
}
|
|
finally
|
|
{
|
|
DataAccess.EndImportSession();
|
|
DataAccess.RestoreImportModePragmas();
|
|
}
|
|
Console.WriteLine($"Total posts added: {postsAdded}");
|
|
}
|
|
else
|
|
{
|
|
switch (args[0])
|
|
{
|
|
case "-?":
|
|
case "-h":
|
|
case "--help":
|
|
PrintHelp();
|
|
break;
|
|
|
|
case "-V":
|
|
case "--version":
|
|
Console.WriteLine(System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown");
|
|
break;
|
|
|
|
case "--parse":
|
|
string blogNameToParse = args[1];
|
|
int postsAdded = 0;
|
|
try
|
|
{
|
|
DataAccess.EnableImportModePragmas();
|
|
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse, startFromBlogName, logTraversalRecordImports);
|
|
}
|
|
finally
|
|
{
|
|
DataAccess.RestoreImportModePragmas();
|
|
}
|
|
Console.WriteLine($"Total posts added: {postsAdded}");
|
|
break;
|
|
|
|
case "--test":
|
|
Console.WriteLine("Test command not implemented");
|
|
break;
|
|
|
|
case "--post":
|
|
TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
|
|
break;
|
|
|
|
case "--posts": //write post's blogs to file
|
|
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
|
|
break;
|
|
|
|
case "--blogs": //write blogs to file
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
|
|
break;
|
|
|
|
case "--collect": //collect notes from all posts
|
|
bool withoutNotesOnly = true;
|
|
DateTime? beforeDate = DateTime.Now;
|
|
bool explicitDateSupplied = false;
|
|
|
|
if (args.Length < 2)
|
|
{
|
|
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
|
|
// Parse withoutNotesOnly flag
|
|
if (args.Length > 1 && args[1] is not null)
|
|
{
|
|
if (args[1] == "1")
|
|
{
|
|
withoutNotesOnly = true;
|
|
Console.WriteLine("Parsed");
|
|
}
|
|
else
|
|
{
|
|
withoutNotesOnly = false;
|
|
Console.WriteLine("--NOT Parsed");
|
|
}
|
|
|
|
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
|
|
}
|
|
|
|
// Parse optional beforeDate parameter
|
|
if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
|
|
{
|
|
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
|
{
|
|
beforeDate = parsedDate;
|
|
explicitDateSupplied = true;
|
|
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
|
|
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
|
|
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
|
|
bool managedCollectRun = false;
|
|
if (!withoutNotesOnly && !explicitDateSupplied)
|
|
{
|
|
DataAccess.EnsureCollectRunStateTableExists();
|
|
var runState = DataAccess.GetCollectRunState();
|
|
if (runState != null && !runState.Value.complete)
|
|
{
|
|
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
|
|
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
|
|
}
|
|
else
|
|
{
|
|
beforeDate = DateTime.Now;
|
|
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
|
|
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
|
|
}
|
|
managedCollectRun = true;
|
|
}
|
|
|
|
exitCode = CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
|
|
break;
|
|
|
|
case "--blogsR": //collect notes from all posts
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
|
|
break;
|
|
|
|
case "--blogsO": //collect notes from all posts
|
|
int from = 1, to = 999999, top = 100;
|
|
|
|
if (args[1] is not null && args[2] is not null && args[3] is not null)
|
|
{
|
|
from = int.Parse(args[1]);
|
|
to = int.Parse(args[2]);
|
|
top = int.Parse(args[3]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Expected FROM TO--");
|
|
exitCode = 2;
|
|
}
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
|
break;
|
|
|
|
case "--bop": //collect notes from all posts
|
|
from = 1; to = 999999; top = 100;
|
|
|
|
if (args[1] is not null && args[2] is not null && args[3] is not null)
|
|
{
|
|
from = int.Parse(args[1]);
|
|
to = int.Parse(args[2]);
|
|
top = int.Parse(args[3]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Expected FROM TO--");
|
|
exitCode = 2;
|
|
}
|
|
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
|
break;
|
|
|
|
case "--replies": //update reply text
|
|
CollectMissingReplyText().GetAwaiter().GetResult();
|
|
break;
|
|
|
|
case "--likes":
|
|
string likeBlog = args.Length > 1 ? args[1] : null;
|
|
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
|
|
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
|
|
break;
|
|
|
|
case "--urldump":
|
|
DumpUrls(settings.GetValue<string>("PathOutputUrls"));
|
|
break;
|
|
|
|
case "--ingest":
|
|
exitCode = IngestMode.Run(config, args.Skip(1).ToArray());
|
|
break;
|
|
|
|
case "--output":
|
|
exitCode = OutputMode.Run(config);
|
|
break;
|
|
|
|
case "--revert":
|
|
exitCode = RevertMode.Run(config, args.Length > 1 ? args[1] : null);
|
|
break;
|
|
|
|
case "--correct":
|
|
{
|
|
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
|
var correctArgs = args.Skip(1)
|
|
.Where(a => !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase))
|
|
.ToArray();
|
|
exitCode = CorrectMode.Run(config, correctArgs, applyChanges);
|
|
break;
|
|
}
|
|
|
|
case "--updatepaths":
|
|
{
|
|
string rootPath = args.Length > 1 ? args[1] : (settings.GetValue<string>("PathTTRoot") ?? settings.GetValue<string>("PathInput") ?? string.Empty);
|
|
exitCode = UpdateBlogPathsRunner.Run(rootPath);
|
|
break;
|
|
}
|
|
|
|
case "--importposts":
|
|
{
|
|
if (args.Length < 2)
|
|
{
|
|
Console.WriteLine("Usage: --importposts <path-to-legacy-posts.db>");
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
exitCode = LegacyPostsDbImporter.Run(args[1]);
|
|
break;
|
|
}
|
|
|
|
default:
|
|
Console.WriteLine("** Unknown Command ** " + args[0]);
|
|
exitCode = 2;
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
System.Console.WriteLine("<fin>:/");
|
|
//System.Console.ReadKey();
|
|
return exitCode;
|
|
}
|
|
|
|
static void PrintHelp()
|
|
{
|
|
Console.WriteLine("\t Parse .txt files to find blogs");
|
|
|
|
Console.WriteLine("-?, -h, --help\t Usage help");
|
|
|
|
Console.WriteLine("-V, --version\t Print the application version");
|
|
|
|
Console.WriteLine("--\t End of options: treat every following token as a literal operand");
|
|
|
|
Console.WriteLine("--parse\t Parse .txt files with specified blogname");
|
|
|
|
Console.WriteLine("--test\t Calls API for given blogname and postID");
|
|
|
|
Console.WriteLine("--post\t Traverse the input directory tree checking .txt files for corruption");
|
|
|
|
Console.WriteLine("--posts\t For each Post in DB, write blogname to file");
|
|
|
|
Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");
|
|
|
|
Console.WriteLine("--collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
|
|
|
|
Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");
|
|
|
|
Console.WriteLine("--blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
|
|
|
|
Console.WriteLine("--bop [from] [to] [top]\t Write ALL blog names to file, limited by FROM TO TOP range arguments");
|
|
|
|
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
|
|
|
|
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
|
|
|
|
Console.WriteLine("--force\t (with --likes) Ignore cooldown and refresh every fully-backfilled blog");
|
|
|
|
Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
|
|
|
|
Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json");
|
|
|
|
Console.WriteLine("--api4\t Use TumblrApi4 settings from appsettings.json");
|
|
|
|
Console.WriteLine("--start [blogname]\t Start traversal alphabetically at this blog name");
|
|
|
|
Console.WriteLine("--api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
|
|
|
|
Console.WriteLine("--ingest [blogname]\t Ingest Tumblr .txt exports from appSettings:PathTTRoot into TL.db (all blogs, or single blog if name given)");
|
|
|
|
Console.WriteLine("--output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
|
|
|
|
Console.WriteLine("--revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
|
|
|
|
Console.WriteLine("--correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
|
|
|
Console.WriteLine("--correct --apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
|
|
|
Console.WriteLine("--updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from <root>\\Index and set Blogs.TTFolderPath");
|
|
|
|
Console.WriteLine("--importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine("Exit status: 0 = success; 1 = unexpected error; 2 = usage error (unknown command or bad/missing arguments); 3 = incomplete (--collect paused on a rate limit, or skipped posts after transient API failures) - relaunch to resume");
|
|
}
|
|
|
|
static void WritePostBlogsToFile(string outPath)
|
|
{
|
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
List<string> blogs = new List<string>();
|
|
foreach (var post in posts)
|
|
{
|
|
blogs.Add(post.Item1);
|
|
}
|
|
blogs = blogs.Distinct().ToList();
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
|
|
{
|
|
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
DataAccess.UpdateBlogOutput(blog);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
|
|
{
|
|
List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
DataAccess.UpdateBlogOutput(blog);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
|
|
{
|
|
List<Tuple<string, long>> posts = DataAccess.GetReplies();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var post in posts)
|
|
{
|
|
Console.WriteLine(@"https://{0}.tumblr.com/post/{1}", post.Item1, post.Item2);
|
|
sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void DumpUrls(string? outPath)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(outPath))
|
|
{
|
|
Console.WriteLine("--PathOutputUrls is not configured in appsettings.json--");
|
|
return;
|
|
}
|
|
|
|
Console.WriteLine($"Starting URL extraction to {outPath}...");
|
|
HashSet<string> uniqueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
Regex urlRegex = new Regex(@"https?://[^\s""'<>]+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
|
|
|
int rowsProcessed = 0;
|
|
foreach (var texts in DataAccess.GetAllPostTextColumns())
|
|
{
|
|
rowsProcessed++;
|
|
if (rowsProcessed % 10000 == 0)
|
|
{
|
|
Console.WriteLine($"Scanned {rowsProcessed} rows... found {uniqueUrls.Count} unique URLs so far.");
|
|
}
|
|
|
|
foreach (var text in texts)
|
|
{
|
|
var matches = urlRegex.Matches(text);
|
|
foreach (Match match in matches)
|
|
{
|
|
uniqueUrls.Add(match.Value);
|
|
}
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"Scan complete. Sorting and saving {uniqueUrls.Count} distinct URLs...");
|
|
var sortedUrls = uniqueUrls.ToList();
|
|
sortedUrls.Sort();
|
|
|
|
File.WriteAllLines(outPath, sortedUrls);
|
|
Console.WriteLine($"Saved URLs to {outPath}");
|
|
}
|
|
|
|
protected static bool ContainsAny(string input, List<string> contains)
|
|
{
|
|
if (string.IsNullOrEmpty(input)) return false;
|
|
|
|
|
|
foreach (string item in contains)
|
|
{
|
|
if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
protected static string NormalizeBlogFolderName(string folderName)
|
|
{
|
|
return folderName
|
|
.Replace("_1", "")
|
|
.Replace("_2", "")
|
|
.Replace("_3", "")
|
|
.Replace("_4", "")
|
|
.Replace("_5", "")
|
|
.Replace("_6", "")
|
|
.Replace("_7", "")
|
|
.Replace("_8", "")
|
|
.Replace("_9", "");
|
|
}
|
|
|
|
static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
|
|
{
|
|
const int MaxPages = 10;
|
|
try
|
|
{
|
|
Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}.tumblr.com/post/{postID}");
|
|
|
|
int replyCount = 0;
|
|
int rowsUpdated = 0;
|
|
int emptyReplyCount = 0;
|
|
long pageTimestamp = 0;
|
|
int page = 0;
|
|
bool sawHealthyResponse = false;
|
|
|
|
while (page < MaxPages)
|
|
{
|
|
page++;
|
|
await Task.Delay(2000);
|
|
var key = ApiKeyPool.GetCurrentKey();
|
|
var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);
|
|
|
|
if (postsResponse?.statusCode == "TooManyRequests")
|
|
{
|
|
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
|
|
ApiKeyPool.MarkRateLimited(key, retry);
|
|
Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
|
|
return;
|
|
}
|
|
|
|
if (postsResponse?.meta?.status != 429)
|
|
ApiKeyPool.MarkAvailable(key);
|
|
|
|
if (postsResponse?.meta?.status == 200)
|
|
sawHealthyResponse = true;
|
|
|
|
if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
|
|
{
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"[Reply Text] No notes returned for {blogName}.tumblr.com/post/{postID} on page {page} (before_timestamp={pageTimestamp})");
|
|
Console.ForegroundColor = prevColor;
|
|
Console.WriteLine($"[Reply Text] meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
|
|
var raw = postsResponse?.rawJson ?? string.Empty;
|
|
if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
|
|
Console.WriteLine($"[Reply Text] raw: {raw}");
|
|
|
|
bool isNotFound = postsResponse?.meta?.status == 404
|
|
|| string.Equals(postsResponse?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
|
|
if (isNotFound)
|
|
{
|
|
var notFoundColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"[Reply Text] Post {blogName}.tumblr.com/post/{postID} returned 404 - marking NotFound=1 and replies '?'");
|
|
Console.ForegroundColor = notFoundColor;
|
|
DataAccess.UpdatePostMarkNotFound(blogName, postID);
|
|
rowsUpdated += DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
|
|
emptyReplyCount += rowsUpdated;
|
|
}
|
|
break;
|
|
}
|
|
|
|
long lastNoteTimestamp = 0;
|
|
foreach (var note in postsResponse.response.notes)
|
|
{
|
|
if (note.timestamp > 0)
|
|
lastNoteTimestamp = note.timestamp;
|
|
|
|
if (note.type == "reply")
|
|
{
|
|
if (!string.IsNullOrEmpty(note.reply_text))
|
|
{
|
|
rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
|
|
replyCount++;
|
|
|
|
string displayText = note.reply_text.Length > 100
|
|
? note.reply_text.Substring(0, 100) + "..."
|
|
: note.reply_text;
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine($" [{note.blog_name}] {displayText}");
|
|
Console.ForegroundColor = previousColor;
|
|
}
|
|
else
|
|
{
|
|
rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
|
|
emptyReplyCount++;
|
|
Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
|
|
}
|
|
}
|
|
else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
|
|
{
|
|
rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
|
|
replyCount++;
|
|
|
|
string displayText = note.reply_text.Length > 100
|
|
? note.reply_text.Substring(0, 100) + "..."
|
|
: note.reply_text;
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Cyan;
|
|
Console.WriteLine($" [{note.blog_name}] {displayText} (reblog comment)");
|
|
Console.ForegroundColor = previousColor;
|
|
}
|
|
}
|
|
|
|
if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
|
|
{
|
|
break;
|
|
}
|
|
pageTimestamp = lastNoteTimestamp;
|
|
}
|
|
|
|
// If pagination finished without updating any reply rows but the API responded healthily
|
|
// at least once, mark the post's outstanding '.' replies '?' so the work queue releases it.
|
|
if (rowsUpdated == 0 && sawHealthyResponse)
|
|
{
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"[Reply Text] Marking outstanding replies for {blogName}.tumblr.com/post/{postID} '?' (healthy 200 OK, no matching reply notes after {page} page(s))");
|
|
Console.ForegroundColor = prevColor;
|
|
rowsUpdated = DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
|
|
emptyReplyCount += rowsUpdated;
|
|
}
|
|
|
|
Console.WriteLine($"[Reply Text] Done {blogName}.tumblr.com/post/{postID}: notes={replyCount}, rowsUpdated={rowsUpdated}, empty='?'={emptyReplyCount}, pages={page}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}.tumblr.com/post/{postID}: {ex.Message}");
|
|
Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
|
|
}
|
|
}
|
|
|
|
static async Task CollectMissingReplyText()
|
|
{
|
|
try
|
|
{
|
|
// Ensure the replyText column exists
|
|
DataAccess.EnsureReplyTextColumnExists();
|
|
|
|
Console.WriteLine("Starting collection of missing reply text...");
|
|
Console.WriteLine("Processing replies directly from Tumblr API.");
|
|
Console.WriteLine();
|
|
|
|
int totalProcessedCount = 0;
|
|
int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
|
|
|
Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
|
|
Console.WriteLine();
|
|
|
|
while (true)
|
|
{
|
|
// Re-query each iteration so posts whose replies got filled in as a side-effect
|
|
// of a previous post's update are skipped without burning an API call.
|
|
var batch = DataAccess.GetRepliesWithFilledText(limit: 1);
|
|
|
|
if (batch is null || batch.Count == 0)
|
|
{
|
|
Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
|
|
break;
|
|
}
|
|
|
|
var reply = batch[0];
|
|
var blogName = reply.Item1;
|
|
var postID = reply.Item2;
|
|
var timestamp = reply.Item3;
|
|
|
|
await FetchAndStoreReplyText(blogName, postID, timestamp);
|
|
totalProcessedCount++;
|
|
|
|
int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
|
|
double completionPct = initialTotal > 0
|
|
? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
|
|
: 100.0;
|
|
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Cyan;
|
|
Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
|
|
Console.ForegroundColor = previousColor;
|
|
|
|
if (totalProcessedCount % 50 == 0)
|
|
{
|
|
await Task.Delay(2000);
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Reply Text] Error: {ex.Message}");
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
}
|
|
|
|
static ConsoleColor GetMatchedCounterColor(long matchedCount)
|
|
{
|
|
if (matchedCount <= 0)
|
|
return ConsoleColor.White;
|
|
|
|
return ((matchedCount - 1) % 3) switch
|
|
{
|
|
0 => ConsoleColor.Red,
|
|
1 => ConsoleColor.Green,
|
|
_ => ConsoleColor.Blue
|
|
};
|
|
}
|
|
|
|
static void WriteLikesTotalsLine(string blogName, string label, long parsedForBlog, long matchedForBlog, int likedCountForBlog)
|
|
{
|
|
if (likedCountForBlog > 0)
|
|
{
|
|
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
|
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
|
|
|
Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: ");
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
|
|
Console.Write(matchedForBlog);
|
|
Console.ForegroundColor = previousColor;
|
|
Console.WriteLine($"/{likedCountForBlog} ({matchedPct:F2}%)");
|
|
}
|
|
else
|
|
{
|
|
Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog} | Matched: ");
|
|
var previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
|
|
Console.Write(matchedForBlog);
|
|
Console.ForegroundColor = previousColor;
|
|
Console.WriteLine();
|
|
}
|
|
}
|
|
|
|
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
|
|
{
|
|
try
|
|
{
|
|
DataAccess.EnsureBlogsLikesColumnsExist();
|
|
|
|
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
|
|
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
|
|
|
|
if (blogsToProcess.Count == 0)
|
|
{
|
|
Console.WriteLine("No blogs found to process likes for.");
|
|
return;
|
|
}
|
|
|
|
int backfillCount = blogsToProcess.Count(b => b.Item2 == 0);
|
|
int refreshCount = blogsToProcess.Count(b => b.Item2 == 1);
|
|
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh).");
|
|
|
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
|
{
|
|
// 1/sec average, matching --collect: the CDN reacts to aggregate traffic from the IP,
|
|
// not to per-command rates.
|
|
PermitLimit = 60,
|
|
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
|
QueueLimit = 1,
|
|
Window = TimeSpan.FromMinutes(1),
|
|
SegmentsPerWindow = 60,
|
|
AutoReplenishment = true
|
|
});
|
|
|
|
for (int i = 0; i < blogsToProcess.Count; i++)
|
|
{
|
|
var blogInfo = blogsToProcess[i];
|
|
int remaining = blogsToProcess.Count - i - 1;
|
|
string blogName = blogInfo.Item1;
|
|
int likesPulled = blogInfo.Item2;
|
|
long cursor = blogInfo.Item3;
|
|
long storedNewestTs = blogInfo.Item4;
|
|
|
|
bool isRefresh = likesPulled == 1;
|
|
long parsedForBlog = 0;
|
|
long matchedForBlog = 0;
|
|
int likedCountForBlog = 0;
|
|
long observedMaxLikedTs = storedNewestTs;
|
|
int newInsertedInRefresh = 0;
|
|
|
|
// Refresh always starts from the top (newest) and walks backward until it crosses
|
|
// the stored high-water mark. Backfill resumes from its last persisted cursor.
|
|
if (isRefresh) cursor = 0;
|
|
|
|
string mode = isRefresh ? "REFRESH" : "BACKFILL";
|
|
Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
|
|
|
|
bool hasMoreLikes = true;
|
|
bool isFirstPage = true;
|
|
|
|
while (hasMoreLikes)
|
|
{
|
|
// Wait for a permit rather than giving up on one: the limiter paces the loop, it is
|
|
// not a failure condition. Only one acquire is ever pending, so QueueLimit = 1 suffices.
|
|
using RateLimitLease lease = await limiter.AcquireAsync(1);
|
|
if (!lease.IsAcquired)
|
|
{
|
|
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
|
return;
|
|
}
|
|
|
|
var key = ApiKeyPool.GetCurrentKey();
|
|
var response = await APIAccess.GrabLikes(key, blogName, cursor);
|
|
|
|
if (response?.statusCode == "TooManyRequests")
|
|
{
|
|
int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
|
|
ApiKeyPool.MarkRateLimited(key, retry);
|
|
|
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
|
continue;
|
|
}
|
|
|
|
if (response.meta?.status != 429)
|
|
ApiKeyPool.MarkAvailable(key);
|
|
|
|
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
|
|
if (isRefresh)
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
else
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
|
|
break;
|
|
}
|
|
|
|
if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
|
|
{
|
|
Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
|
|
if (isRefresh)
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
else
|
|
{
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
|
|
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
|
|
}
|
|
hasMoreLikes = false;
|
|
break;
|
|
}
|
|
|
|
if (response.response.liked_count > 0)
|
|
likedCountForBlog = response.response.liked_count;
|
|
|
|
Console.WriteLine($"[Likes] [Fetched {response.response.liked_posts.Count}] {blogName}");
|
|
|
|
bool crossedHighWaterMark = false;
|
|
|
|
foreach (var post in response.response.liked_posts)
|
|
{
|
|
parsedForBlog++;
|
|
|
|
long postID = 0;
|
|
try { postID = Convert.ToInt64(post.id); } catch { continue; }
|
|
|
|
// liked_timestamp is when the user liked the post (matches the `before` cursor semantics).
|
|
// It's the only reliable field for the refresh stop condition.
|
|
long likedTs = 0;
|
|
try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { }
|
|
|
|
if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs)
|
|
{
|
|
Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh.");
|
|
crossedHighWaterMark = true;
|
|
break;
|
|
}
|
|
|
|
if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs;
|
|
|
|
string authorBlog = post.blog_name?.ToString() ?? ".";
|
|
string postURL = post.post_url?.ToString() ?? ".";
|
|
string date = post.date?.ToString() ?? ".";
|
|
|
|
// Legacy format fields
|
|
string body = post.body?.ToString() ?? ".";
|
|
string summary = post.summary?.ToString() ?? ".";
|
|
string slug = post.slug?.ToString() ?? ".";
|
|
string reblogURL = post.source_url?.ToString() ?? ".";
|
|
string reblogName = post.reblogged_from_name?.ToString() ?? post.source_title?.ToString() ?? ".";
|
|
string rootBlogName = post.reblogged_root_name?.ToString() ?? ".";
|
|
string rootURL = post.reblogged_root_url?.ToString() ?? ".";
|
|
|
|
// Quick check for tags
|
|
string tags = ".";
|
|
if (post.tags != null)
|
|
{
|
|
try { tags = string.Join(", ", post.tags); }
|
|
catch { }
|
|
}
|
|
|
|
bool hasImage = false;
|
|
string photoURL = ".";
|
|
string photoCaption = ".";
|
|
|
|
if (post.photos != null)
|
|
{
|
|
hasImage = true;
|
|
try
|
|
{
|
|
var firstPhoto = post.photos[0];
|
|
if (firstPhoto != null)
|
|
{
|
|
if (firstPhoto.original_size != null)
|
|
photoURL = firstPhoto.original_size.url?.ToString() ?? ".";
|
|
photoCaption = firstPhoto.caption?.ToString() ?? ".";
|
|
}
|
|
} catch { }
|
|
}
|
|
else if (body.IndexOf("<img", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
hasImage = true;
|
|
}
|
|
|
|
// Optional fields
|
|
string quote = ".";
|
|
string audioCaption = ".";
|
|
string question = ".";
|
|
string answer = ".";
|
|
string title = post.title?.ToString() ?? ".";
|
|
string downloadedFiles = ".";
|
|
string reblogKey = post.reblog_key?.ToString() ?? ".";
|
|
string link = ".";
|
|
|
|
// Only insert if any of the data contains strings from ContainsList
|
|
bool shouldInsert = false;
|
|
string matchedFieldName = string.Empty;
|
|
|
|
// Let's check a few fields that usually have URLs or relevant info that might match ContainsList
|
|
var fieldsToCheck = new (string Name, string Value)[] {
|
|
("postURL", postURL),
|
|
("authorBlog", authorBlog),
|
|
("reblogURL", reblogURL),
|
|
("reblogName", reblogName),
|
|
("rootBlogName", rootBlogName),
|
|
("rootURL", rootURL),
|
|
("summary", summary),
|
|
("body", body),
|
|
("tags", tags),
|
|
("photoURL", photoURL),
|
|
("photoCaption", photoCaption),
|
|
("quote", quote),
|
|
("question", question),
|
|
("answer", answer),
|
|
("downloadedFiles", downloadedFiles),
|
|
("slug", slug)
|
|
};
|
|
|
|
foreach (var field in fieldsToCheck)
|
|
{
|
|
if (ContainsAny(field.Value, contains))
|
|
{
|
|
shouldInsert = true;
|
|
matchedFieldName = field.Name;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (shouldInsert)
|
|
{
|
|
matchedForBlog++;
|
|
if (isRefresh) newInsertedInRefresh++;
|
|
Console.WriteLine($"[Likes] [Matched] {blogName} | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
|
|
await Task.Delay(3000);
|
|
DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
|
|
reblogName, summary, quote, body, tags, link, photoURL,
|
|
photoCaption, downloadedFiles, audioCaption, question, answer,
|
|
title, hasImage, true, rootBlogName: rootBlogName, rootURL: rootURL);
|
|
}
|
|
}
|
|
|
|
WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
|
|
|
// Branch B: refresh terminates as soon as we crossed the high-water mark.
|
|
if (isRefresh && crossedHighWaterMark)
|
|
{
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
hasMoreLikes = false;
|
|
break;
|
|
}
|
|
|
|
// Branch A: on the very first page, capture & persist the newest liked_timestamp
|
|
// so subsequent refresh runs (after backfill completes) have a stopping point.
|
|
if (!isRefresh && isFirstPage && observedMaxLikedTs > 0)
|
|
{
|
|
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
|
|
}
|
|
isFirstPage = false;
|
|
|
|
// Determine the next BeforeCursor.
|
|
long nextCursor = 0;
|
|
if (response.response._links?.next?.query_params != null)
|
|
{
|
|
long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
|
|
}
|
|
|
|
if (!isRefresh)
|
|
{
|
|
// Persist cursor progress after every page so resume is always up-to-date
|
|
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
|
|
}
|
|
|
|
if (nextCursor > 0)
|
|
{
|
|
cursor = nextCursor;
|
|
Console.WriteLine($"[Likes] Pagination Next Cursor: {cursor}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
|
|
if (isRefresh)
|
|
{
|
|
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
|
|
}
|
|
else
|
|
{
|
|
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
|
|
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist);
|
|
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
|
|
}
|
|
hasMoreLikes = false;
|
|
}
|
|
|
|
Console.WriteLine($"[{remaining} remaining] Done with {blogName}");
|
|
await Task.Delay(1000); // 1-second delay between pages
|
|
}
|
|
|
|
if (isRefresh)
|
|
Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}");
|
|
WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
|
}
|
|
|
|
Console.WriteLine("[Likes] [Likes collection complete]");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error collecting likes: {ex.Message}");
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
}
|
|
|
|
// Backoff between in-place retries of a transient infrastructure failure. Most CDN 403s and edge
|
|
// 5xxs clear within a few seconds, so retrying here saves the post its single attempt for the pass.
|
|
static readonly int[] TransientBackoffSeconds = { 1, 4, 10 };
|
|
|
|
// Fetches one page, retrying transient failures in place. Rate limits are returned to the caller
|
|
// untouched — those are handled by pausing the whole run, not by retrying this post.
|
|
static async Task<Root> FetchNotesPage(Tuple<string, long, long, long> post, string beforeTimestamp)
|
|
{
|
|
Root response = null!;
|
|
|
|
for (int attempt = 0; ; attempt++)
|
|
{
|
|
var key = ApiKeyPool.GetCurrentKey();
|
|
response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
|
|
|
|
if (response.statusCode == "TooManyRequests")
|
|
{
|
|
ApiKeyPool.MarkRateLimited(key, response.retryInSeconds > 0 ? response.retryInSeconds : 60);
|
|
return response;
|
|
}
|
|
|
|
if (!response.transientFailure)
|
|
{
|
|
// Only a response that actually reached the API says anything about the key's standing.
|
|
ApiKeyPool.MarkAvailable(key);
|
|
return response;
|
|
}
|
|
|
|
if (attempt >= TransientBackoffSeconds.Length)
|
|
return response;
|
|
|
|
int delay = TransientBackoffSeconds[attempt];
|
|
Console.WriteLine($"[Transient] retry {attempt + 1}/{TransientBackoffSeconds.Length} in {delay}s");
|
|
await Task.Delay(delay * 1000);
|
|
}
|
|
}
|
|
|
|
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
|
|
{
|
|
try
|
|
{
|
|
static bool IsNotFound(Root? r)
|
|
{
|
|
return (r?.meta != null && r.meta.status == 404) || string.Equals(r?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
int APICount = DataAccess.GetAPICount();
|
|
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
|
|
|
|
var allNotes = new List<dynamic>();
|
|
int page = 1;
|
|
string beforeTimestamp = post.Item3.ToString();
|
|
bool hasReplies = false;
|
|
const int maxPages = 500;
|
|
var response = await FetchNotesPage(post, beforeTimestamp);
|
|
|
|
if (response.statusCode == "TooManyRequests")
|
|
return "TooManyRequests";
|
|
|
|
if (response.transientFailure)
|
|
{
|
|
Console.WriteLine($"[Skip] {post.Item1}/{post.Item2} — {response.statusCode} after {TransientBackoffSeconds.Length} retries");
|
|
return "Transient";
|
|
}
|
|
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2}");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
Thread.Sleep(1000);
|
|
return "NotFound";
|
|
}
|
|
|
|
// Pagination loop
|
|
while (true)
|
|
{
|
|
int noteCount = response?.response?.notes?.Count ?? 0;
|
|
Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");
|
|
|
|
if (response?.response == null)
|
|
{
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (empty response payload)");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return "NotFound";
|
|
}
|
|
Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
|
|
return "FAILURE";
|
|
}
|
|
if (response.response.notes == null)
|
|
{
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (notes payload missing)");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return "NotFound";
|
|
}
|
|
Console.WriteLine("##### Notes Null - WHY? ###");
|
|
return "FAILURE";
|
|
}
|
|
|
|
// Accumulate notes
|
|
allNotes.AddRange(response.response.notes);
|
|
|
|
// Pagination: check for next
|
|
var nextLink = response.response._links?.next;
|
|
if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
|
|
{
|
|
Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
|
|
break;
|
|
}
|
|
|
|
beforeTimestamp = nextLink.query_params.before_timestamp;
|
|
page++;
|
|
if (page > maxPages)
|
|
{
|
|
Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}.tumblr.com/post/{post.Item2}. Aborting further pagination.");
|
|
break;
|
|
}
|
|
|
|
response = await FetchNotesPage(post, beforeTimestamp);
|
|
if (response.statusCode == "TooManyRequests")
|
|
return "TooManyRequests";
|
|
|
|
if (response.transientFailure)
|
|
{
|
|
Console.WriteLine($"[Skip] {post.Item1}/{post.Item2} — {response.statusCode} on page {page} after {TransientBackoffSeconds.Length} retries");
|
|
return "Transient";
|
|
}
|
|
|
|
if (IsNotFound(response))
|
|
{
|
|
Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}.tumblr.com/post/{post.Item2}");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return "NotFound";
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
|
|
|
|
// Process all accumulated notes
|
|
foreach (var note in allNotes)
|
|
{
|
|
note.reblog_parent_blog_name = post.Item1;
|
|
note.post_id = post.Item2.ToString();
|
|
if (note.type == "reply")
|
|
hasReplies = true;
|
|
DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
|
|
}
|
|
|
|
return "Success";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
return "UNKNOWN";
|
|
}
|
|
|
|
// Consecutive transient failures that mean the API edge is rejecting traffic wholesale rather than
|
|
// blipping on one post. Past this, skipping post-by-post would just hammer a closed door.
|
|
const int MaxConsecutiveTransient = 10;
|
|
|
|
static async Task<int> CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
|
|
{
|
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
|
|
|
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
|
|
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
|
|
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
|
|
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
|
|
|
|
int skipped = 0;
|
|
int consecutiveTransient = 0;
|
|
|
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
|
{
|
|
// 1/sec average. Sustained higher rates draw CDN-level 403s that the API's own rate-limit
|
|
// headers never warn about, so this sits well under the per-key quota on purpose.
|
|
PermitLimit = 60,
|
|
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
|
QueueLimit = 1,
|
|
Window = TimeSpan.FromMinutes(1),
|
|
SegmentsPerWindow = 60,
|
|
AutoReplenishment = true
|
|
});
|
|
|
|
try
|
|
{
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
while (posts.Count > 0)
|
|
{
|
|
// First post not yet attempted this process. If all remaining have been attempted, the pass
|
|
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
|
|
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
|
|
if (post == null)
|
|
{
|
|
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
|
|
break;
|
|
}
|
|
|
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
|
|
|
// Wait for a permit rather than giving up on one: the limiter paces the loop, it is
|
|
// not a failure condition. Only one acquire is ever pending, so QueueLimit = 1 suffices.
|
|
using RateLimitLease lease = await limiter.AcquireAsync(1);
|
|
if (!lease.IsAcquired)
|
|
{
|
|
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
|
return 3; // abort without completing the run so a later launch resumes
|
|
}
|
|
|
|
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
|
|
string status = await GrabNotes(post);
|
|
|
|
if (status == "Transient")
|
|
{
|
|
// Retries in GrabNotes are already exhausted. Skip the post so the pass can make
|
|
// progress; it stays unmarked in the DB, so the next launch picks it up again.
|
|
attempted.Add((post.Item1, post.Item2));
|
|
skipped++;
|
|
consecutiveTransient++;
|
|
|
|
if (consecutiveTransient >= MaxConsecutiveTransient)
|
|
{
|
|
Console.WriteLine($"[Abort] {consecutiveTransient} consecutive transient failures - the API edge is rejecting traffic. Pausing run; relaunch to resume. ({skipped} post(s) skipped)");
|
|
return 3;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
consecutiveTransient = 0;
|
|
|
|
if (status == "Success")
|
|
{
|
|
attempted.Add((post.Item1, post.Item2));
|
|
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
|
}
|
|
else if (status == "NotFound")
|
|
{
|
|
attempted.Add((post.Item1, post.Item2));
|
|
Console.WriteLine("GrabNotes Result: NotFound");
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
}
|
|
else if (status == "TooManyRequests")
|
|
{
|
|
// Throttle, not a real per-post failure: don't consume this post's single attempt.
|
|
// Abort the pass without completing so a later launch resumes against the same cutoff.
|
|
Console.WriteLine($"GrabNotes Result: TooManyRequests - pausing run; relaunch to resume. ({skipped} post(s) skipped)");
|
|
return 3;
|
|
}
|
|
else
|
|
{
|
|
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
|
|
attempted.Add((post.Item1, post.Item2));
|
|
Console.WriteLine("GrabNotes Result: " + status);
|
|
}
|
|
}
|
|
|
|
// Re-fetch the updated list after processing the current post
|
|
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
|
}
|
|
}
|
|
|
|
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
|
|
if (managedRun)
|
|
{
|
|
DataAccess.CompleteCollectRun();
|
|
Console.WriteLine("Full re-check run complete.");
|
|
}
|
|
|
|
if (skipped > 0)
|
|
{
|
|
Console.WriteLine($"Pass finished with {skipped} post(s) skipped after transient failures; relaunch to retry them.");
|
|
return 3;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
|
|
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
|
|
{
|
|
DateTime directoryStart = DateTime.Now;
|
|
int directoryRecordsImported = 0;
|
|
Console.WriteLine($"[Directory Start] {path} | {directoryStart:yyyy-MM-dd HH:mm:ss.fff}");
|
|
|
|
try
|
|
{
|
|
// Get all directories in the current directory and sort them alphabetically
|
|
var directories = Directory.GetDirectories(path);
|
|
Array.Sort(directories, StringComparer.InvariantCulture);
|
|
|
|
foreach (var directory in directories)
|
|
{
|
|
Console.WriteLine("Directory: " + directory);
|
|
TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName, startFromBlogName, logRecordImports); // Recursively traverse subdirectories
|
|
}
|
|
|
|
try
|
|
{
|
|
bool headerWasWritten = false;
|
|
// Process all files in the current directory
|
|
foreach (var file in Directory.GetFiles(path))
|
|
{
|
|
string normalizedDirectoryName = NormalizeBlogFolderName(new DirectoryInfo(path).Name);
|
|
bool isAtOrAfterStart = string.IsNullOrWhiteSpace(startFromBlogName) || string.Compare(normalizedDirectoryName, startFromBlogName, StringComparison.OrdinalIgnoreCase) >= 0;
|
|
|
|
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)
|
|
&& (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)
|
|
&& isAtOrAfterStart)
|
|
{
|
|
try
|
|
{
|
|
var urls = new List<string>();
|
|
var reblog = new ReblogRecord();
|
|
|
|
foreach (string line in File.ReadLines(file))
|
|
{
|
|
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
|
{
|
|
if (!reblog.reblogURL.Contains("deactivated")
|
|
&& reblog.reblogURL.Length != 0
|
|
&& ContainsAny(reblog.reblogURL, contains))
|
|
{
|
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
if (!headerWasWritten)
|
|
{
|
|
headerWasWritten = true;
|
|
}
|
|
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
|
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
|
|
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
reblog.title, false);
|
|
recordImportStopwatch.Stop();
|
|
|
|
postsAdded++;
|
|
directoryRecordsImported++;
|
|
if (logRecordImports)
|
|
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
|
|
|
// Output hyperlink and post date
|
|
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
|
}
|
|
}
|
|
reblog = new ReblogRecord();
|
|
reblog.postID = line.Substring(9).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
//reblog = new ReblogRecord();
|
|
|
|
reblog.reblogURL = line.Substring(12).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.reblogName = line.Substring(13).Trim();
|
|
}
|
|
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.downloadedFiles = line.Substring(17).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.reblogKey = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.date = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.body = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.postURL = line.Substring(10).Trim();
|
|
}
|
|
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.answer = line.Substring(8).Trim();
|
|
}
|
|
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.audioCaption = line.Substring(15).Trim();
|
|
}
|
|
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.blogName = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.link = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.photoCaption = line.Substring(15).Trim();
|
|
}
|
|
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.photoURL = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.question = line.Substring(10).Trim();
|
|
}
|
|
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.quote = line.Substring(7).Trim();
|
|
}
|
|
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.slug = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.summary = line.Substring(9).Trim();
|
|
}
|
|
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.tags = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
reblog.title = line.Substring(7).Trim();
|
|
}
|
|
|
|
if ((reblog.downloadedFiles != "."
|
|
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
|
{
|
|
if ((ContainsAny(reblog.downloadedFiles, contains)
|
|
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
|
//&& !reblog.reblogURL.Contains("deactivated")
|
|
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
|
{
|
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
if (!headerWasWritten)
|
|
{
|
|
headerWasWritten = true;
|
|
}
|
|
//if (reblog.reblogURL.Contains("/blog/private")
|
|
// || reblog.body.Contains("/blog/private"))
|
|
//{
|
|
// Console.WriteLine(reblog.postID);
|
|
// Console.WriteLine(reblog.postURL);
|
|
// Console.WriteLine(reblog.date);
|
|
// Console.WriteLine(reblog.body);
|
|
// Console.WriteLine(reblog.reblogKey);
|
|
// Console.WriteLine(reblog.reblogURL);
|
|
// Console.WriteLine(reblog.reblogName);
|
|
// Console.WriteLine(reblog.downloadedFiles);
|
|
//}
|
|
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
|
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
|
|
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
reblog.title, true);
|
|
recordImportStopwatch.Stop();
|
|
|
|
postsAdded++;
|
|
directoryRecordsImported++;
|
|
if (logRecordImports)
|
|
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
|
|
|
// Output hyperlink and post date
|
|
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
|
}
|
|
}
|
|
}
|
|
urls.Sort();
|
|
|
|
|
|
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
//{
|
|
// foreach (string line in urls.Distinct())
|
|
// {
|
|
// sw.WriteLine(line);
|
|
// }
|
|
//}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine("The file could not be read:");
|
|
Console.WriteLine(e.Message);
|
|
}
|
|
}
|
|
// Add your file processing logic here
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"An error occurred: {ex.Message}");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
DateTime directoryEnd = DateTime.Now;
|
|
TimeSpan elapsed = directoryEnd - directoryStart;
|
|
Console.WriteLine($"[Directory End] {path} | {directoryEnd:yyyy-MM-dd HH:mm:ss.fff} | Duration: {elapsed:hh\\:mm\\:ss\\.fff} | Records Imported: {directoryRecordsImported}");
|
|
}
|
|
}
|
|
|
|
|
|
static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
|
|
{
|
|
var directories = Directory.GetDirectories(path);
|
|
Array.Sort(directories, StringComparer.InvariantCulture);
|
|
|
|
foreach (var directory in directories)
|
|
{
|
|
//Console.WriteLine("Directory: " + directory);
|
|
TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
|
|
}
|
|
try
|
|
{
|
|
bool headerWasWritten = false;
|
|
foreach (var file in Directory.GetFiles(path))
|
|
{
|
|
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
|
|
{
|
|
try
|
|
{
|
|
var urls = new List<string>();
|
|
var reblog = new ReblogRecord();
|
|
|
|
foreach (string line in File.ReadLines(file))
|
|
{
|
|
if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
|
|
{
|
|
Console.WriteLine(file);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine("The file could not be read:");
|
|
Console.WriteLine(e.Message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"An error occurred: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
}
|