using System.Threading.RateLimiting;
using Microsoft.Extensions.Http.Resilience;
using Microsoft.Extensions;
using Microsoft.Extensions.Configuration;
using System.Configuration;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Text.RegularExpressions;

namespace URLNotesGrabberCORE
{
    internal class Program
    {

        static int Main(string[] args)
        {
            // Reset console color on exit (including Ctrl+C)
            Console.CancelKeyPress += (s, e) => Console.ResetColor();
            AppDomain.CurrentDomain.ProcessExit += (s, e) => Console.ResetColor();

            try
            {
                return Run(args);
            }
            catch (Exception ex)
            {
                // Turn configuration errors / unguarded indexers into a quiet, deterministic exit code.
                Console.Error.WriteLine(ex.Message);
                return 1;
            }
        }

        static int Run(string[] args)
        {
            int exitCode = 0;
            IConfiguration config = new ConfigurationBuilder()
                                    .SetBasePath(Directory.GetCurrentDirectory())
                                    .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
                                    .AddCommandLine(args)
                                    .Build();
            var settings = config.GetSection("appSettings");

            string apiSectionName = "TumblrApi";
            bool apiExplicitlySet = false;
            string startFromBlogName = string.Empty;
            bool forceIgnoreCooldown = false;
            List<string> filteredArgs = new List<string>();
            for (int i = 0; i < args.Length; i++)
            {
                if (args[i] == "--")
                {
                    // POSIX end-of-options: everything after is a literal operand.
                    for (int j = i + 1; j < args.Length; j++) filteredArgs.Add(args[j]);
                    break;
                }

                if (string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
                {
                    forceIgnoreCooldown = true;
                    continue;
                }

                if (string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
                {
                    apiSectionName = "TumblrApi3";
                    apiExplicitlySet = true;
                    continue;
                }

                if (string.Equals(args[i], "--api4", StringComparison.OrdinalIgnoreCase))
                {
                    apiSectionName = "TumblrApi4";
                    apiExplicitlySet = true;
                    continue;
                }

                if (string.Equals(args[i], "--api", StringComparison.OrdinalIgnoreCase))
                {
                    if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
                    {
                        apiSectionName = args[i + 1].Trim();
                        apiExplicitlySet = true;
                        i++;
                    }
                    else
                    {
                        Console.WriteLine("--Missing API section after --api. Using default TumblrApi.--");
                    }
                    continue;
                }

                if (string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
                {
                    if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
                    {
                        startFromBlogName = args[i + 1].Trim();
                        i++;
                    }
                    else
                    {
                        Console.WriteLine("--Missing blog name after --start. Ignoring.--");
                    }
                    continue;
                }

                filteredArgs.Add(args[i]);
            }

            args = filteredArgs.ToArray();

            string? dbPath = config["appSettings:PathDB"];
            if (string.IsNullOrEmpty(dbPath))
                throw new InvalidOperationException("PathDB is not configured in appsettings.json");

            ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null);

            // Setup Dual Logging
            bool enableFileLogging = settings.GetValue("EnableFileLogging", true);
            if (enableFileLogging)
            {
                string logPath = "console_output.log";
                if (File.Exists(logPath))
                {
                    string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
                    string archiveDirectory = "logs";

                    // Ensure the archive directory exists
                    if (!Directory.Exists(archiveDirectory))
                    {
                        Directory.CreateDirectory(archiveDirectory);
                    }

                    string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
                    File.Move(logPath, newPath);
                }

                StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
                DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
                Console.SetOut(dualLogger);
            }

            List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
            bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);

            if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
            {
                int postsAdded = 0;
                try
                {
                    DataAccess.EnableImportModePragmas();
                    DataAccess.BeginImportSession();
                    TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports);
                }
                finally
                {
                    DataAccess.EndImportSession();
                    DataAccess.RestoreImportModePragmas();
                }
                Console.WriteLine($"Total posts added: {postsAdded}");
            }
            else
            {
                switch (args[0])
                {
                    case "-?":
                    case "-h":
                    case "--help":
                        PrintHelp();
                        break;

                    case "-V":
                    case "--version":
                        Console.WriteLine(System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown");
                        break;

                    case "--parse":
                        if (args.Length < 2)
                        {
                            Console.WriteLine("Usage: --parse <blogname>");
                            exitCode = 2;
                            break;
                        }
                        string blogNameToParse = args[1];
                        int postsAdded = 0;
                        try
                        {
                            DataAccess.EnableImportModePragmas();
                            TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse, startFromBlogName, logTraversalRecordImports);
                        }
                        finally
                        {
                            DataAccess.RestoreImportModePragmas();
                        }
                        Console.WriteLine($"Total posts added: {postsAdded}");
                        break;
                    
                    case "--test":
                        Console.WriteLine("Test command not implemented");
                        break;

                    case "--post":
                        TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
                        break;

                    case "--posts": //write post's blogs to file
                        WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
                        break;

                    case "--blogs": //write blogs to file
                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
                        break;

                    case "--collect": //collect notes from all posts
                        bool withoutNotesOnly = true;
                        DateTime? beforeDate = DateTime.Now;
                        bool explicitDateSupplied = false;

                        if (args.Length < 2)
                        {
                            Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
                            exitCode = 2;
                            break;
                        }

                        // Parse withoutNotesOnly flag
                        if (args.Length > 1 && args[1] is not null)
                        {
                            if (args[1] == "1")
                            {
                                withoutNotesOnly = true;
                                Console.WriteLine("Parsed");
                            }
                            else
                            {
                                withoutNotesOnly = false;
                                Console.WriteLine("--NOT Parsed");
                            }

                            Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
                        }

                        // Parse optional beforeDate parameter
                        if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
                        {
                            if (DateTime.TryParse(args[2], out DateTime parsedDate))
                            {
                                beforeDate = parsedDate;
                                explicitDateSupplied = true;
                                Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
                            }
                            else
                            {
                                Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
                                exitCode = 2;
                                break;
                            }
                        }

                        // Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
                        // persist it so an interrupted run resumes against the same cutoff and a completed run stops
                        // instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
                        bool managedCollectRun = false;
                        if (!withoutNotesOnly && !explicitDateSupplied)
                        {
                            DataAccess.EnsureCollectRunStateTableExists();
                            var runState = DataAccess.GetCollectRunState();
                            if (runState != null && !runState.Value.complete)
                            {
                                beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
                                Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
                            }
                            else
                            {
                                beforeDate = DateTime.Now;
                                DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
                                Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
                            }
                            managedCollectRun = true;
                        }

                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
                        break;

                    case "--blogsR": //collect notes from all posts
                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
                        break;

                    case "--blogsO": //collect notes from all posts
                        int from = 1, to = 999999, top = 100;

                        if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
                        {
                            from = int.Parse(args[1]);
                            to = int.Parse(args[2]);
                            top = int.Parse(args[3]);
                        }
                        else
                        {
                            Console.WriteLine("--Expected FROM TO--");
                            exitCode = 2;
                            break;
                        }
                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
                        break;

                    case "--bop": //collect notes from all posts
                        from = 1; to = 999999; top = 100;

                        if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
                        {
                            from = int.Parse(args[1]);
                            to = int.Parse(args[2]);
                            top = int.Parse(args[3]);
                        }
                        else
                        {
                            Console.WriteLine("--Expected FROM TO--");
                            exitCode = 2;
                            break;
                        }
                        WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
                        break;

                    case "--replies": //update reply text
                        CollectMissingReplyText().GetAwaiter().GetResult();
                        break;

                    case "--likes":
                        string likeBlog = args.Length > 1 ? args[1] : null;
                        int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
                        CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
                        break;

                    case "--urldump":
                        DumpUrls(settings.GetValue<string>("PathOutputUrls"));
                        break;

                    case "--ingest":
                        exitCode = IngestMode.Run(config, args.Skip(1).ToArray());
                        break;

                    case "--output":
                        exitCode = OutputMode.Run(config);
                        break;

                    case "--revert":
                        exitCode = RevertMode.Run(config, args.Length > 1 ? args[1] : null);
                        break;

                    case "--correct":
                    {
                        bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
                        var correctArgs = args.Skip(1)
                            .Where(a => !string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase))
                            .ToArray();
                        exitCode = CorrectMode.Run(config, correctArgs, applyChanges);
                        break;
                    }

                    case "--updatepaths":
                    {
                        string rootPath = args.Length > 1 ? args[1] : (settings.GetValue<string>("PathTTRoot") ?? settings.GetValue<string>("PathInput") ?? string.Empty);
                        exitCode = UpdateBlogPathsRunner.Run(rootPath);
                        break;
                    }

                    case "--importposts":
                    {
                        if (args.Length < 2)
                        {
                            Console.WriteLine("Usage: --importposts <path-to-legacy-posts.db>");
                            exitCode = 2;
                            break;
                        }
                        exitCode = LegacyPostsDbImporter.Run(args[1]);
                        break;
                    }

                    default:
                        Console.WriteLine("** Unknown Command ** " + args[0]);
                        exitCode = 2;
                        break;
                }
            }



            System.Console.WriteLine("<fin>:/");
            //System.Console.ReadKey();
            return exitCode;
        }

        static void PrintHelp()
        {
            Console.WriteLine("\t Parse .txt files to find blogs");

            Console.WriteLine("-?, -h, --help\t Usage help");

            Console.WriteLine("-V, --version\t Print the application version");

            Console.WriteLine("--\t End of options: treat every following token as a literal operand");

            Console.WriteLine("--parse\t Parse .txt files with specified blogname");

            Console.WriteLine("--test\t Calls API for given blogname and postID");

            Console.WriteLine("--post\t Traverse the input directory tree checking .txt files for corruption");

            Console.WriteLine("--posts\t For each Post in DB, write blogname to file");

            Console.WriteLine("--blogs\t For each Blog in DB, write blogname to file");

            Console.WriteLine("--collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");

            Console.WriteLine("--blogsR\t For each Note that is a REPLY, write blogname to file ");

            Console.WriteLine("--blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");

            Console.WriteLine("--bop [from] [to] [top]\t Write ALL blog names to file, limited by FROM TO TOP range arguments");

            Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");

            Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");

            Console.WriteLine("--force\t (with --likes) Ignore cooldown and refresh every fully-backfilled blog");

            Console.WriteLine("--urldump\t Scan all posts' text columns and extract suspected URLs to configured file");

            Console.WriteLine("--api3\t Use TumblrApi3 settings from appsettings.json");

            Console.WriteLine("--api4\t Use TumblrApi4 settings from appsettings.json");

            Console.WriteLine("--start [blogname]\t Start traversal alphabetically at this blog name");

            Console.WriteLine("--api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");

            Console.WriteLine("--ingest [blogname]\t Ingest Tumblr .txt exports from appSettings:PathTTRoot into TL.db (all blogs, or single blog if name given)");

            Console.WriteLine("--output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");

            Console.WriteLine("--revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");

            Console.WriteLine("--correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");

            Console.WriteLine("--correct --apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");

            Console.WriteLine("--updatepaths [rootPath]\t Read .tumblr/.tmblrpriv metadata from <root>\\Index and set Blogs.TTFolderPath");

            Console.WriteLine("--importposts [path-to-posts.db]\t One-time migration: copy legacy ThreeTxtFileHelper posts.db rows into TL.db");

            Console.WriteLine();
            Console.WriteLine("Exit status: 0 = success; 1 = unexpected error; 2 = usage error (unknown command or bad/missing arguments)");
        }

        static void WritePostBlogsToFile(string outPath)
        {
            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                List<string> blogs = new List<string>();
                foreach (var post in posts)
                {
                    blogs.Add(post.Item1);
                }
                blogs = blogs.Distinct().ToList();
                blogs.Sort();
                blogs.Reverse();

                foreach (var blog in blogs)
                {
                    Console.WriteLine(blog);
                    sw.WriteLine(blog + ".tumblr.com");
                }
            }
        }

        static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
        {
            List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
            blogs.Sort();
            blogs.Reverse();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                foreach (var blog in blogs)
                {
                    Console.WriteLine(blog);
                    sw.WriteLine(blog + ".tumblr.com");
                    DataAccess.UpdateBlogOutput(blog);
                }
            }
        }

        static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
        {
            List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
            blogs.Sort();
            blogs.Reverse();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                foreach (var blog in blogs)
                {
                    Console.WriteLine(blog);
                    sw.WriteLine(blog + ".tumblr.com");
                    DataAccess.UpdateBlogOutput(blog);
                }
            }
        }

        static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
        {
            List<Tuple<string, long>> posts = DataAccess.GetReplies();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                foreach (var post in posts)
                {
                    Console.WriteLine(@"https://{0}.tumblr.com/post/{1}", post.Item1, post.Item2);
                    sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
                }
            }
        }

        static void DumpUrls(string? outPath)
        {
            if (string.IsNullOrWhiteSpace(outPath))
            {
                Console.WriteLine("--PathOutputUrls is not configured in appsettings.json--");
                return;
            }

            Console.WriteLine($"Starting URL extraction to {outPath}...");
            HashSet<string> uniqueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
            Regex urlRegex = new Regex(@"https?://[^\s""'<>]+", RegexOptions.IgnoreCase | RegexOptions.Compiled);

            int rowsProcessed = 0;
            foreach (var texts in DataAccess.GetAllPostTextColumns())
            {
                rowsProcessed++;
                if (rowsProcessed % 10000 == 0)
                {
                    Console.WriteLine($"Scanned {rowsProcessed} rows... found {uniqueUrls.Count} unique URLs so far.");
                }

                foreach (var text in texts)
                {
                    var matches = urlRegex.Matches(text);
                    foreach (Match match in matches)
                    {
                        uniqueUrls.Add(match.Value);
                    }
                }
            }

            Console.WriteLine($"Scan complete. Sorting and saving {uniqueUrls.Count} distinct URLs...");
            var sortedUrls = uniqueUrls.ToList();
            sortedUrls.Sort();

            File.WriteAllLines(outPath, sortedUrls);
            Console.WriteLine($"Saved URLs to {outPath}");
        }

        protected static bool ContainsAny(string input, List<string> contains)
        {
            if (string.IsNullOrEmpty(input)) return false;


            foreach (string item in contains)
            {
                if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
                {
                    return true;
                }
            }
            return false;
        }

        protected static string NormalizeBlogFolderName(string folderName)
        {
            return folderName
                .Replace("_1", "")
                .Replace("_2", "")
                .Replace("_3", "")
                .Replace("_4", "")
                .Replace("_5", "")
                .Replace("_6", "")
                .Replace("_7", "")
                .Replace("_8", "")
                .Replace("_9", "");
        }

        static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
        {
            const int MaxPages = 10;
            try
            {
                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}.tumblr.com/post/{postID}");

                int replyCount = 0;
                int rowsUpdated = 0;
                int emptyReplyCount = 0;
                long pageTimestamp = 0;
                int page = 0;
                bool sawHealthyResponse = false;

                while (page < MaxPages)
                {
                    page++;
                    await Task.Delay(2000);
                    var key = ApiKeyPool.GetCurrentKey();
                    var postsResponse = await APIAccess.GrabPostWithReplies(key, blogName, postID, pageTimestamp);

                    if (postsResponse?.statusCode == "TooManyRequests")
                    {
                        int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
                        ApiKeyPool.MarkRateLimited(key, retry);
                        Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
                        return;
                    }

                    if (postsResponse?.meta?.status != 429)
                        ApiKeyPool.MarkAvailable(key);

                    if (postsResponse?.meta?.status == 200)
                        sawHealthyResponse = true;

                    if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
                    {
                        var prevColor = Console.ForegroundColor;
                        Console.ForegroundColor = ConsoleColor.Yellow;
                        Console.WriteLine($"[Reply Text] No notes returned for {blogName}.tumblr.com/post/{postID} on page {page} (before_timestamp={pageTimestamp})");
                        Console.ForegroundColor = prevColor;
                        Console.WriteLine($"[Reply Text]   meta.status={postsResponse?.meta?.status}, meta.msg=\"{postsResponse?.meta?.msg}\", statusCode={postsResponse?.statusCode ?? "N/A"}");
                        var raw = postsResponse?.rawJson ?? string.Empty;
                        if (raw.Length > 500) raw = raw.Substring(0, 500) + "...[truncated]";
                        Console.WriteLine($"[Reply Text]   raw: {raw}");

                        bool isNotFound = postsResponse?.meta?.status == 404
                            || string.Equals(postsResponse?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
                        if (isNotFound)
                        {
                            var notFoundColor = Console.ForegroundColor;
                            Console.ForegroundColor = ConsoleColor.Yellow;
                            Console.WriteLine($"[Reply Text] Post {blogName}.tumblr.com/post/{postID} returned 404 - marking NotFound=1 and replies '?'");
                            Console.ForegroundColor = notFoundColor;
                            DataAccess.UpdatePostMarkNotFound(blogName, postID);
                            rowsUpdated += DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
                            emptyReplyCount += rowsUpdated;
                        }
                        break;
                    }

                    long lastNoteTimestamp = 0;
                    foreach (var note in postsResponse.response.notes)
                    {
                        if (note.timestamp > 0)
                            lastNoteTimestamp = note.timestamp;

                        if (note.type == "reply")
                        {
                            if (!string.IsNullOrEmpty(note.reply_text))
                            {
                                rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                                replyCount++;

                                string displayText = note.reply_text.Length > 100
                                    ? note.reply_text.Substring(0, 100) + "..."
                                    : note.reply_text;
                                var previousColor = Console.ForegroundColor;
                                Console.ForegroundColor = ConsoleColor.Green;
                                Console.WriteLine($"  [{note.blog_name}] {displayText}");
                                Console.ForegroundColor = previousColor;
                            }
                            else
                            {
                                rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, "?");
                                emptyReplyCount++;
                                Console.WriteLine($"[Reply Text] Reply from {note.blog_name} returned with empty reply_text - marked '?'");
                            }
                        }
                        else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
                        {
                            rowsUpdated += DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                            replyCount++;

                            string displayText = note.reply_text.Length > 100
                                ? note.reply_text.Substring(0, 100) + "..."
                                : note.reply_text;
                            var previousColor = Console.ForegroundColor;
                            Console.ForegroundColor = ConsoleColor.Cyan;
                            Console.WriteLine($"  [{note.blog_name}] {displayText} (reblog comment)");
                            Console.ForegroundColor = previousColor;
                        }
                    }

                    if (lastNoteTimestamp <= 0 || (pageTimestamp > 0 && lastNoteTimestamp >= pageTimestamp))
                    {
                        break;
                    }
                    pageTimestamp = lastNoteTimestamp;
                }

                // If pagination finished without updating any reply rows but the API responded healthily
                // at least once, mark the post's outstanding '.' replies '?' so the work queue releases it.
                if (rowsUpdated == 0 && sawHealthyResponse)
                {
                    var prevColor = Console.ForegroundColor;
                    Console.ForegroundColor = ConsoleColor.Yellow;
                    Console.WriteLine($"[Reply Text] Marking outstanding replies for {blogName}.tumblr.com/post/{postID} '?' (healthy 200 OK, no matching reply notes after {page} page(s))");
                    Console.ForegroundColor = prevColor;
                    rowsUpdated = DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
                    emptyReplyCount += rowsUpdated;
                }

                Console.WriteLine($"[Reply Text] Done {blogName}.tumblr.com/post/{postID}: notes={replyCount}, rowsUpdated={rowsUpdated}, empty='?'={emptyReplyCount}, pages={page}");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}.tumblr.com/post/{postID}: {ex.Message}");
                Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
            }
        }

        static async Task CollectMissingReplyText()
        {
            try
            {
                // Ensure the replyText column exists
                DataAccess.EnsureReplyTextColumnExists();

                Console.WriteLine("Starting collection of missing reply text...");
                Console.WriteLine("Processing replies directly from Tumblr API.");
                Console.WriteLine();

                int totalProcessedCount = 0;
                int initialTotal = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;

                Console.WriteLine($"[Reply Text] Total replies to process: {initialTotal}");
                Console.WriteLine();

                while (true)
                {
                    // Re-query each iteration so posts whose replies got filled in as a side-effect
                    // of a previous post's update are skipped without burning an API call.
                    var batch = DataAccess.GetRepliesWithFilledText(limit: 1);

                    if (batch is null || batch.Count == 0)
                    {
                        Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
                        break;
                    }

                    var reply = batch[0];
                    var blogName = reply.Item1;
                    var postID = reply.Item2;
                    var timestamp = reply.Item3;

                    await FetchAndStoreReplyText(blogName, postID, timestamp);
                    totalProcessedCount++;

                    int remainingNow = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
                    double completionPct = initialTotal > 0
                        ? ((initialTotal - remainingNow) / (double)initialTotal) * 100.0
                        : 100.0;

                    var previousColor = Console.ForegroundColor;
                    Console.ForegroundColor = ConsoleColor.Cyan;
                    Console.WriteLine($"[{remainingNow} remaining] [processed={totalProcessedCount}, initial={initialTotal}] ({completionPct:F1}%)");
                    Console.ForegroundColor = previousColor;

                    if (totalProcessedCount % 50 == 0)
                    {
                        await Task.Delay(2000);
                    }
                }

                Console.WriteLine($"[Reply Text] Complete. Total API-fetched posts: {totalProcessedCount}.");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"[Reply Text] Error: {ex.Message}");
                Console.WriteLine(ex.ToString());
            }
        }

        static ConsoleColor GetMatchedCounterColor(long matchedCount)
        {
            if (matchedCount <= 0)
                return ConsoleColor.White;

            return ((matchedCount - 1) % 3) switch
            {
                0 => ConsoleColor.Red,
                1 => ConsoleColor.Green,
                _ => ConsoleColor.Blue
            };
        }

        static void WriteLikesTotalsLine(string blogName, string label, long parsedForBlog, long matchedForBlog, int likedCountForBlog)
        {
            if (likedCountForBlog > 0)
            {
                double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
                double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;

                Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: ");
                var previousColor = Console.ForegroundColor;
                Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
                Console.Write(matchedForBlog);
                Console.ForegroundColor = previousColor;
                Console.WriteLine($"/{likedCountForBlog} ({matchedPct:F2}%)");
            }
            else
            {
                Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog} | Matched: ");
                var previousColor = Console.ForegroundColor;
                Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
                Console.Write(matchedForBlog);
                Console.ForegroundColor = previousColor;
                Console.WriteLine();
            }
        }

        static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
        {
            try
            {
                DataAccess.EnsureBlogsLikesColumnsExist();

                Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
                var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);

                if (blogsToProcess.Count == 0)
                {
                    Console.WriteLine("No blogs found to process likes for.");
                    return;
                }

                int backfillCount = blogsToProcess.Count(b => b.Item2 == 0);
                int refreshCount = blogsToProcess.Count(b => b.Item2 == 1);
                Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh).");

                RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
                {
                    PermitLimit = 300,
                    QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                    QueueLimit = 1,
                    Window = TimeSpan.FromMinutes(1),
                    SegmentsPerWindow = 60,
                    AutoReplenishment = true
                });

                for (int i = 0; i < blogsToProcess.Count; i++)
                {
                    var blogInfo = blogsToProcess[i];
                    int remaining = blogsToProcess.Count - i - 1;
                    string blogName = blogInfo.Item1;
                    int likesPulled = blogInfo.Item2;
                    long cursor = blogInfo.Item3;
                    long storedNewestTs = blogInfo.Item4;

                    bool isRefresh = likesPulled == 1;
                    long parsedForBlog = 0;
                    long matchedForBlog = 0;
                    int likedCountForBlog = 0;
                    long observedMaxLikedTs = storedNewestTs;
                    int newInsertedInRefresh = 0;

                    // Refresh always starts from the top (newest) and walks backward until it crosses
                    // the stored high-water mark. Backfill resumes from its last persisted cursor.
                    if (isRefresh) cursor = 0;

                    string mode = isRefresh ? "REFRESH" : "BACKFILL";
                    Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");

                    bool hasMoreLikes = true;
                    bool isFirstPage = true;

                    while (hasMoreLikes)
                    {
                        using RateLimitLease lease = limiter.AttemptAcquire(1);
                        if (!lease.IsAcquired)
                        {
                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                            return;
                        }

                        var key = ApiKeyPool.GetCurrentKey();
                        var response = await APIAccess.GrabLikes(key, blogName, cursor);

                        if (response?.statusCode == "TooManyRequests")
                        {
                            int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                            ApiKeyPool.MarkRateLimited(key, retry);

                            ApiKeyPool.SleepUntilAnyAvailable(30);
                            continue;
                        }

                        if (response.meta?.status != 429)
                            ApiKeyPool.MarkAvailable(key);

                        if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
                        {
                            Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
                            if (isRefresh)
                                DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
                            else
                                DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
                            break;
                        }

                        if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
                        {
                            Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
                            if (isRefresh)
                                DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
                            else
                            {
                                DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
                                DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
                            }
                            hasMoreLikes = false;
                            break;
                        }

                        if (response.response.liked_count > 0)
                            likedCountForBlog = response.response.liked_count;

                        Console.WriteLine($"[Likes] [Fetched {response.response.liked_posts.Count}] {blogName}");

                        bool crossedHighWaterMark = false;

                        foreach (var post in response.response.liked_posts)
                        {
                            parsedForBlog++;

                            long postID = 0;
                            try { postID = Convert.ToInt64(post.id); } catch { continue; }

                            // liked_timestamp is when the user liked the post (matches the `before` cursor semantics).
                            // It's the only reliable field for the refresh stop condition.
                            long likedTs = 0;
                            try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { }

                            if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs)
                            {
                                Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh.");
                                crossedHighWaterMark = true;
                                break;
                            }

                            if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs;

                            string authorBlog = post.blog_name?.ToString() ?? ".";
                            string postURL = post.post_url?.ToString() ?? ".";
                            string date = post.date?.ToString() ?? ".";

                            // Legacy format fields
                            string body = post.body?.ToString() ?? ".";
                            string summary = post.summary?.ToString() ?? ".";
                            string slug = post.slug?.ToString() ?? ".";
                            string reblogURL = post.source_url?.ToString() ?? ".";
                            string reblogName = post.reblogged_from_name?.ToString() ?? post.source_title?.ToString() ?? ".";
                            string rootBlogName = post.reblogged_root_name?.ToString() ?? ".";
                            string rootURL = post.reblogged_root_url?.ToString() ?? ".";

                            // Quick check for tags
                            string tags = ".";
                            if (post.tags != null)
                            {
                                try { tags = string.Join(", ", post.tags); }
                                catch { }
                            }

                            bool hasImage = false;
                            string photoURL = ".";
                            string photoCaption = ".";

                            if (post.photos != null)
                            {
                                hasImage = true;
                                try
                                {
                                    var firstPhoto = post.photos[0];
                                    if (firstPhoto != null)
                                    {
                                        if (firstPhoto.original_size != null)
                                            photoURL = firstPhoto.original_size.url?.ToString() ?? ".";
                                        photoCaption = firstPhoto.caption?.ToString() ?? ".";
                                    }
                                } catch { }
                            }
                            else if (body.IndexOf("<img", StringComparison.OrdinalIgnoreCase) >= 0)
                            {
                                hasImage = true;
                            }

                            // Optional fields
                            string quote = ".";
                            string audioCaption = ".";
                            string question = ".";
                            string answer = ".";
                            string title = post.title?.ToString() ?? ".";
                            string downloadedFiles = ".";
                            string reblogKey = post.reblog_key?.ToString() ?? ".";
                            string link = ".";

                            // Only insert if any of the data contains strings from ContainsList
                            bool shouldInsert = false;
                            string matchedFieldName = string.Empty;

                            // Let's check a few fields that usually have URLs or relevant info that might match ContainsList
                            var fieldsToCheck = new (string Name, string Value)[] {
                                ("postURL", postURL),
                                ("authorBlog", authorBlog),
                                ("reblogURL", reblogURL),
                                ("reblogName", reblogName),
                                ("rootBlogName", rootBlogName),
                                ("rootURL", rootURL),
                                ("summary", summary),
                                ("body", body),
                                ("tags", tags),
                                ("photoURL", photoURL),
                                ("photoCaption", photoCaption),
                                ("quote", quote),
                                ("question", question),
                                ("answer", answer),
                                ("downloadedFiles", downloadedFiles),
                                ("slug", slug)
                            };

                            foreach (var field in fieldsToCheck)
                            {
                                if (ContainsAny(field.Value, contains))
                                {
                                    shouldInsert = true;
                                    matchedFieldName = field.Name;
                                    break;
                                }
                            }

if (shouldInsert)
{
                                matchedForBlog++;
                                if (isRefresh) newInsertedInRefresh++;
                                Console.WriteLine($"[Likes] [Matched] {blogName} | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
                                await Task.Delay(3000);
                                DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
                                                        reblogName, summary, quote, body, tags, link, photoURL,
                                                        photoCaption, downloadedFiles, audioCaption, question, answer,
                                                        title, hasImage, true, rootBlogName: rootBlogName, rootURL: rootURL);
                            }
                        }

                        WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);

                        // Branch B: refresh terminates as soon as we crossed the high-water mark.
                        if (isRefresh && crossedHighWaterMark)
                        {
                            DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
                            hasMoreLikes = false;
                            break;
                        }

                        // Branch A: on the very first page, capture & persist the newest liked_timestamp
                        // so subsequent refresh runs (after backfill completes) have a stopping point.
                        if (!isRefresh && isFirstPage && observedMaxLikedTs > 0)
                        {
                            DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
                        }
                        isFirstPage = false;

                        // Determine the next BeforeCursor.
                        long nextCursor = 0;
                        if (response.response._links?.next?.query_params != null)
                        {
                            long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
                        }

                        if (!isRefresh)
                        {
                            // Persist cursor progress after every page so resume is always up-to-date
                            long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
                            DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
                        }

                        if (nextCursor > 0)
                        {
                            cursor = nextCursor;
                            Console.WriteLine($"[Likes] Pagination Next Cursor: {cursor}");
                        }
                        else
                        {
                            Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
                            if (isRefresh)
                            {
                                DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
                            }
                            else
                            {
                                long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
                                DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist);
                                DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
                            }
                            hasMoreLikes = false;
                        }

                        Console.WriteLine($"[{remaining} remaining] Done with {blogName}");
                        await Task.Delay(1000); // 1-second delay between pages
                    }

                    if (isRefresh)
                        Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}");
                    WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
                }

                Console.WriteLine("[Likes] [Likes collection complete]");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error collecting likes: {ex.Message}");
                Console.WriteLine(ex.ToString());
            }
        }

        static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
        {
            try
            {
                static bool IsNotFound(Root? r)
                {
                    return (r?.meta != null && r.meta.status == 404) || string.Equals(r?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
                }

                int APICount = DataAccess.GetAPICount();
                Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");

                var allNotes = new List<dynamic>();
                int page = 1;
                string beforeTimestamp = post.Item3.ToString();
                bool hasReplies = false;
                const int maxPages = 500;
                var key = ApiKeyPool.GetCurrentKey();
                var response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);

                if (response.statusCode == "TooManyRequests")
                {
                    int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                    ApiKeyPool.MarkRateLimited(key, retry);
                    return "TooManyRequests";
                }

                if (response.meta?.status != 429)
                    ApiKeyPool.MarkAvailable(key);

                if (IsNotFound(response))
                {
                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2}");
                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                    Thread.Sleep(1000);
                    return "NotFound";
                }
                if (response == null)
                {
                    Console.WriteLine("##### Response is null - API Failure? ###");
                    return "FAILURE";
                }
                if (response.statusCode == "TooManyRequests")
                {
                    int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                    ApiKeyPool.MarkRateLimited(key, retry);

                    ApiKeyPool.SleepUntilAnyAvailable(30);
                    return response.statusCode;
                }

                // Pagination loop
                while (true)
                {
                    int noteCount = response?.response?.notes?.Count ?? 0;
                    Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");

                    if (response?.response == null)
                    {
                        if (IsNotFound(response))
                        {
                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (empty response payload)");
                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                            return "NotFound";
                        }
                        Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
                        return "FAILURE";
                    }
                    if (response.response.notes == null)
                    {
                        if (IsNotFound(response))
                        {
                            Console.WriteLine($"API returned 404 Not Found for {post.Item1}.tumblr.com/post/{post.Item2} (notes payload missing)");
                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                            return "NotFound";
                        }
                        Console.WriteLine("##### Notes Null - WHY? ###");
                        return "FAILURE";
                    }

                    // Accumulate notes
                    allNotes.AddRange(response.response.notes);

                    // Pagination: check for next
                    var nextLink = response.response._links?.next;
                    if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
                    {
                        Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
                        break;
                    }

                    beforeTimestamp = nextLink.query_params.before_timestamp;
                    page++;
                    if (page > maxPages)
                    {
                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}.tumblr.com/post/{post.Item2}. Aborting further pagination.");
                        break;
                    }

                    key = ApiKeyPool.GetCurrentKey();
                    response = await APIAccess.GrabNotes(key, post.Item1, post.Item2, beforeTimestamp);
                    if (response.statusCode == "TooManyRequests")
                    {
                        int retry = response.retryInSeconds > 0 ? response.retryInSeconds : 60;
                        ApiKeyPool.MarkRateLimited(key, retry);
                        return "TooManyRequests";
                    }
                    if (response.meta?.status != 429)
                        ApiKeyPool.MarkAvailable(key);

                    if (IsNotFound(response))
                    {
                        Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}.tumblr.com/post/{post.Item2}");
                        DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                        return "NotFound";
                    }
                }

                Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");

                // Process all accumulated notes
                foreach (var note in allNotes)
                {
                    note.reblog_parent_blog_name = post.Item1;
                    note.post_id = post.Item2.ToString();
                    if (note.type == "reply")
                        hasReplies = true;
                    DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
                }

                return "Success";
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
            return "UNKNOWN";
        }

        static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
        {
            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);

            // Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
            // attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
            // post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
            HashSet<(string, long)> attempted = new HashSet<(string, long)>();

            RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
            {
                PermitLimit = 300,
                QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                QueueLimit = 1,
                Window = TimeSpan.FromMinutes(1),
                SegmentsPerWindow = 60,
                AutoReplenishment = true
            });

            try
            {
                using (StreamWriter sw = new StreamWriter(outPath, true))
                {
                    while (posts.Count > 0)
                    {
                        // First post not yet attempted this process. If all remaining have been attempted, the pass
                        // is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
                        var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
                        if (post == null)
                        {
                            Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
                            break;
                        }

                        ApiKeyPool.SleepUntilAnyAvailable(30);

                        string status;

                        using RateLimitLease lease = limiter.AttemptAcquire(1);
                        if (lease.IsAcquired)
                        {
                            Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
                            status = await GrabNotes(post);
                        }
                        else
                        {
                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                            return; // throttle: abort without completing the run so a later launch resumes
                        }

                        if (status == "Success")
                        {
                            attempted.Add((post.Item1, post.Item2));
                            DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
                        }
                        else if (status == "NotFound")
                        {
                            attempted.Add((post.Item1, post.Item2));
                            Console.WriteLine("GrabNotes Result: NotFound");
                            DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                        }
                        else if (status == "TooManyRequests")
                        {
                            // Throttle, not a real per-post failure: don't consume this post's single attempt.
                            // Abort the pass without completing so a later launch resumes against the same cutoff.
                            Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
                            return;
                        }
                        else
                        {
                            // FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
                            attempted.Add((post.Item1, post.Item2));
                            Console.WriteLine("GrabNotes Result: " + status);
                        }

                        // Re-fetch the updated list after processing the current post
                        posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
                    }
                }

                // Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
                if (managedRun)
                {
                    DataAccess.CompleteCollectRun();
                    Console.WriteLine("Full re-check run complete.");
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }


        static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
        {
            DateTime directoryStart = DateTime.Now;
            int directoryRecordsImported = 0;
            Console.WriteLine($"[Directory Start] {path} | {directoryStart:yyyy-MM-dd HH:mm:ss.fff}");

            try
            {
                // Get all directories in the current directory and sort them alphabetically
                var directories = Directory.GetDirectories(path);
                Array.Sort(directories, StringComparer.InvariantCulture);

                foreach (var directory in directories)
                {
                    Console.WriteLine("Directory: " + directory);
                    TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName, startFromBlogName, logRecordImports); // Recursively traverse subdirectories
                }

                try
                {
                    bool headerWasWritten = false;
                    // Process all files in the current directory
                    foreach (var file in Directory.GetFiles(path))
                    {
                        string normalizedDirectoryName = NormalizeBlogFolderName(new DirectoryInfo(path).Name);
                        bool isAtOrAfterStart = string.IsNullOrWhiteSpace(startFromBlogName) || string.Compare(normalizedDirectoryName, startFromBlogName, StringComparison.OrdinalIgnoreCase) >= 0;

                        if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)
                            && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)
                            && isAtOrAfterStart)
                        {
                            try
                            {
                                var urls = new List<string>();
                                var reblog = new ReblogRecord();

                                foreach (string line in File.ReadLines(file))
                                {
                                    if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
                                        {
                                            if (!reblog.reblogURL.Contains("deactivated")
                                                && reblog.reblogURL.Length != 0
                                                && ContainsAny(reblog.reblogURL, contains))
                                            {
                                                DirectoryInfo currentDir = new DirectoryInfo(path);
                                                if (!headerWasWritten)
                                                {
                                                    headerWasWritten = true;
                                                }
                                                string curDir = NormalizeBlogFolderName(currentDir.Name);
                                                urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));

                                                var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
                                                DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                    reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                    reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                    reblog.title, false);
                                                recordImportStopwatch.Stop();

                                                postsAdded++;
                                                directoryRecordsImported++;
                                                if (logRecordImports)
                                                    Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");

                                                // Output hyperlink and post date
                                                //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                            }
                                        }
                                        reblog = new ReblogRecord();
                                        reblog.postID = line.Substring(9).Trim();
                                    }
                                    if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        //reblog = new ReblogRecord();

                                        reblog.reblogURL = line.Substring(12).Trim();
                                    }
                                    if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.reblogName = line.Substring(13).Trim();
                                    }
                                    if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.downloadedFiles = line.Substring(17).Trim();
                                    }
                                    if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.reblogKey = line.Substring(11).Trim();
                                    }
                                    if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.date = line.Substring(6).Trim();
                                    }
                                    if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.body = line.Substring(6).Trim();
                                    }
                                    if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.postURL = line.Substring(10).Trim();
                                    }
                                    if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.answer = line.Substring(8).Trim();
                                    }
                                    if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.audioCaption = line.Substring(15).Trim();
                                    }
                                    if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.blogName = line.Substring(11).Trim();
                                    }
                                    if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.link = line.Substring(6).Trim();
                                    }
                                    if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.photoCaption = line.Substring(15).Trim();
                                    }
                                    if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.photoURL = line.Substring(11).Trim();
                                    }
                                    if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.question = line.Substring(10).Trim();
                                    }
                                    if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.quote = line.Substring(7).Trim();
                                    }
                                    if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.slug = line.Substring(6).Trim();
                                    }
                                    if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.summary = line.Substring(9).Trim();
                                    }
                                    if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.tags = line.Substring(6).Trim();
                                    }
                                    if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
                                    {
                                        reblog.title = line.Substring(7).Trim();
                                    }

                                    if ((reblog.downloadedFiles != "."
                                            || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
                                            || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
                                    {
                                        if ((ContainsAny(reblog.downloadedFiles, contains)
                                            || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
                                            || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
                                            //&& !reblog.reblogURL.Contains("deactivated")
                                            || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
                                        {
                                            DirectoryInfo currentDir = new DirectoryInfo(path);
                                            if (!headerWasWritten)
                                            {
                                                headerWasWritten = true;
                                            }
                                            //if (reblog.reblogURL.Contains("/blog/private")
                                            //    || reblog.body.Contains("/blog/private"))
                                            //{
                                            //    Console.WriteLine(reblog.postID);
                                            //    Console.WriteLine(reblog.postURL);
                                            //    Console.WriteLine(reblog.date);
                                            //    Console.WriteLine(reblog.body);
                                            //    Console.WriteLine(reblog.reblogKey);
                                            //    Console.WriteLine(reblog.reblogURL);
                                            //    Console.WriteLine(reblog.reblogName);
                                            //    Console.WriteLine(reblog.downloadedFiles);
                                            //}
                                            string curDir = NormalizeBlogFolderName(currentDir.Name);
                                            urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));

                                            var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
                                            DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                    reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                    reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                    reblog.title, true);
                                            recordImportStopwatch.Stop();

                                            postsAdded++;
                                            directoryRecordsImported++;
                                            if (logRecordImports)
                                                Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");

                                            // Output hyperlink and post date
                                            //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                        }
                                    }
                                }
                                urls.Sort();


                                //using (StreamWriter sw = new StreamWriter(outPath, true))
                                //{
                                //    foreach (string line in urls.Distinct())
                                //    {
                                //        sw.WriteLine(line);
                                //    }
                                //}
                            }
                            catch (Exception e)
                            {
                                Console.WriteLine("The file could not be read:");
                                Console.WriteLine(e.Message);
                            }
                        }
                        // Add your file processing logic here
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"An error occurred: {ex.Message}");
                }
            }
            finally
            {
                DateTime directoryEnd = DateTime.Now;
                TimeSpan elapsed = directoryEnd - directoryStart;
                Console.WriteLine($"[Directory End] {path} | {directoryEnd:yyyy-MM-dd HH:mm:ss.fff} | Duration: {elapsed:hh\\:mm\\:ss\\.fff} | Records Imported: {directoryRecordsImported}");
            }
        }


        static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
        {
            var directories = Directory.GetDirectories(path);
            Array.Sort(directories, StringComparer.InvariantCulture);

            foreach (var directory in directories)
            {
                //Console.WriteLine("Directory: " + directory);
                TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
            }
            try
            {
                bool headerWasWritten = false;
                foreach (var file in Directory.GetFiles(path))
                {
                    if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
                    {
                        try
                        {
                            var urls = new List<string>();
                            var reblog = new ReblogRecord();

                            foreach (string line in File.ReadLines(file))
                            {
                                if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
                                {
                                    Console.WriteLine(file);
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            Console.WriteLine("The file could not be read:");
                            Console.WriteLine(e.Message);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"An error occurred: {ex.Message}");
            }
        }
    }
}
