using System.Threading.RateLimiting;
using Microsoft.Extensions.Http.Resilience;
using Microsoft.Extensions;
using Microsoft.Extensions.Configuration;
using System.Configuration;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;

namespace URLNotesGrabberCORE
{
    internal class Program
    {

        static void Main(string[] args)
        {
            IConfiguration config = new ConfigurationBuilder()
                                    .SetBasePath(Directory.GetCurrentDirectory())
                                    .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
                                    .AddCommandLine(args)
                                    .Build();
            var settings = config.GetSection("appSettings");

            // Setup Dual Logging
            string logPath = "console_output.log";
            if (File.Exists(logPath))
            {
                string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
                string archiveDirectory = "logs";

                // Ensure the archive directory exists
                if (!Directory.Exists(archiveDirectory))
                {
                    Directory.CreateDirectory(archiveDirectory);
                }

                string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
                File.Move(logPath, newPath);
            }

            StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
            DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
            Console.SetOut(dualLogger);

            List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();

            if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
            {
                int postsAdded = 0;
                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded);
                Console.WriteLine($"Total posts added: {postsAdded}");
            }
            else
            {
                switch (args[0])
                {
                    case "-?":
                        Console.WriteLine("\t Parse .txt files to find blogs");

                        Console.WriteLine("-?\t Usage help");

                        Console.WriteLine("-parse\t Parse .txt files with specified blogname");

                        Console.WriteLine("-test\t Calls API for given blogname and postID");

                        Console.WriteLine("-posts\t For each Post in DB, write blogname to file");

                        Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");

                        Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");

                        Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");

                        Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");

                        Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");

                        break;
                    
                    case "-parse":
                        string blogNameToParse = args[1];
                        int postsAdded = 0;
                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse);
                        Console.WriteLine($"Total posts added: {postsAdded}");
                        break;
                    
                    case "-test":
                        Console.WriteLine("Test command not implemented");
                        break;

                    case "-post":
                        TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
                        break;

                    case "-posts": //write post's blogs to file
                        WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
                        break;

                    case "-blogs": //write blogs to file
                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
                        break;

                    case "-collect": //collect notes from all posts
                        bool withoutNotesOnly = true;
                        DateTime? beforeDate = DateTime.Now;

                        if (args.Length < 2)
                        {
                            Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
                            break;
                        }

                        // Parse withoutNotesOnly flag
                        if (args.Length > 1 && args[1] is not null)
                        {
                            if (args[1] == "1")
                            {
                                withoutNotesOnly = true;
                                Console.WriteLine("Parsed");
                            }
                            else
                            {
                                withoutNotesOnly = false;
                                Console.WriteLine("--NOT Parsed");
                            }

                            Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
                        }

                        // Parse optional beforeDate parameter
                        if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
                        {
                            if (DateTime.TryParse(args[2], out DateTime parsedDate))
                            {
                                beforeDate = parsedDate;
                                Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
                            }
                            else
                            {
                                Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
                                break;
                            }
                        }

                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
                        break;

                    case "-blogsR": //collect notes from all posts
                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
                        break;

                    case "-blogsO": //collect notes from all posts
                        int from = 1, to = 999999, top = 100;

                        if (args[1] is not null && args[2] is not null && args[3] is not null)
                        {
                            from = int.Parse(args[1]);
                            to = int.Parse(args[2]);
                            top = int.Parse(args[3]);
                        }
                        else
                        {
                            Console.WriteLine("--Expected FROM TO--");
                        }
                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
                        break;

                    case "-bop": //collect notes from all posts
                        from = 1; to = 999999; top = 100;

                        if (args[1] is not null && args[2] is not null && args[3] is not null)
                        {
                            from = int.Parse(args[1]);
                            to = int.Parse(args[2]);
                            top = int.Parse(args[3]);
                        }
                        else
                        {
                            Console.WriteLine("--Expected FROM TO--");
                        }
                        WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
                        break;

                    case "-replies": //update reply text
                        CollectMissingReplyText().GetAwaiter().GetResult();
                        break;

                    default:
                        Console.WriteLine("** Unknown Command ** " + args[0]);
                        break;
                }
            }



            System.Console.WriteLine("<fin>:/");
            //System.Console.ReadKey();
        }

        static void WritePostBlogsToFile(string outPath)
        {
            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                List<string> blogs = new List<string>();
                foreach (var post in posts)
                {
                    blogs.Add(post.Item1);
                }
                blogs = blogs.Distinct().ToList();
                blogs.Sort();
                blogs.Reverse();

                foreach (var blog in blogs)
                {
                    Console.WriteLine(blog);
                    sw.WriteLine(blog + ".tumblr.com");
                }
            }
        }

        static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
        {
            List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
            blogs.Sort();
            blogs.Reverse();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                foreach (var blog in blogs)
                {
                    Console.WriteLine(blog);
                    sw.WriteLine(blog + ".tumblr.com");
                    DataAccess.UpdateBlogOutput(blog);
                }
            }
        }

        static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
        {
            List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
            blogs.Sort();
            blogs.Reverse();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                foreach (var blog in blogs)
                {
                    Console.WriteLine(blog);
                    sw.WriteLine(blog + ".tumblr.com");
                    DataAccess.UpdateBlogOutput(blog);
                }
            }
        }

        static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
        {
            List<Tuple<string, long>> posts = DataAccess.GetReplies();

            using (StreamWriter sw = new StreamWriter(outPath, true))
            {
                foreach (var post in posts)
                {
                    Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
                    sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
                }
            }
        }

        protected static bool ContainsAny(string input, List<string> contains)
        {
            if (string.IsNullOrEmpty(input)) return false;


            foreach (string item in contains)
            {
                if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
                {
                    return true;
                }
            }
            return false;
        }

        static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
        {
            try
            {
                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
                // Add 2-second delay before API call to avoid server-side rate limiting
                await Task.Delay(2000);
                var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp);

                if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
                {
                    Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
                    Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
                    Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
                    if (postsResponse?.response != null)
                    {
                        Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
                    }
                    if (!string.IsNullOrEmpty(postsResponse?.rawJson))
                    {
                        Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
                    }
                    
                    // Mark all replies for this post with '?' to indicate API returned no posts
                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
                    DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
                    
                    return;
                }

                // Get the first (and should be only) post
                var post = postsResponse.response.posts.FirstOrDefault();
                if (post?.notes == null)
                {
                    Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
                    return;
                }

                Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");

                // Update each reply with its text
                int replyCount = 0;
                int skippedCount = 0;
                foreach (var note in post.notes)
                {
                    if (note.type == "reply")
                    {
                        if (!string.IsNullOrEmpty(note.reply_text))
                        {
                            DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
                            replyCount++;
                            
                            // Output the reply text being stored
                            string displayText = note.reply_text.Length > 100 
                                ? note.reply_text.Substring(0, 100) + "..." 
                                : note.reply_text;
                            Console.WriteLine($"  [{note.blog_name}] {displayText}");
                        }
                        else
                        {
                            skippedCount++;
                            Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
                        }
                    }
                }

                if (replyCount > 0)
                {
                    Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
                }
                else
                {
                    Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
                }

                // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
                Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
                //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
                //if (cleanupCount > 0)
                //{
                //    Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
                //}
            }
            catch (Exception ex)
            {
                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}");
                Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
            }
        }

        static async Task CollectMissingReplyText()
        {
            try
            {
                // Ensure the replyText column exists
                DataAccess.EnsureReplyTextColumnExists();

                Console.WriteLine("Starting collection of missing reply text...");
                Console.WriteLine("Processing 10 posts at a time.");

                int batchSize = 1;  // Process 10 posts per batch
                int totalProcessedCount = 0;

                RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
                {
                    PermitLimit = 300,
                    QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                    QueueLimit = 1,
                    Window = TimeSpan.FromMinutes(1),
                    SegmentsPerWindow = 60,
                    AutoReplenishment = true
                });

                while (true && totalProcessedCount < batchSize)
                {
                    var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
                    
                    if (postsWithFilledReplies.Count == 0)
                    {
                        Console.WriteLine("No more posts with filled reply text found. Collection complete.");
                        break;
                    }

                    Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
                    int batchProcessedCount = 0;

                    foreach (var post in postsWithFilledReplies)
                    {
                        using RateLimitLease lease = limiter.AttemptAcquire(1);
                        if (lease.IsAcquired)
                        {
                            Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
                            await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
                            batchProcessedCount++;
                            totalProcessedCount++;
                        
                            // Add 1 second delay between attempts
                            await Task.Delay(1000);
                        }
                        else
                        {
                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                            Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
                            return;
                        }
                    }

                    Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
                }

                Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
                Console.WriteLine(ex.ToString());
            }
        }

        static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
        {
            try
            {
                int APICount = DataAccess.GetAPICount();
                Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
                Thread.Sleep(3000);

                var allNotes = new List<dynamic>();
                int page = 1;
                string beforeTimestamp = post.Item3.ToString();
                bool hasReplies = false;
                const int maxPages = 500;
                var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();

                // Handle 404 and error codes
                if (response?.meta != null && response.meta.status == 404)
                {
                    Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)");
                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                    Thread.Sleep(1000);
                    return "NotFound";
                }
                if (response == null)
                {
                    Console.WriteLine("##### Response is null - API Failure? ###");
                    return "FAILURE";
                }
                if (response.statusCode == "NotFound")
                {
                    Thread.Sleep(1000);
                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                    return response.statusCode;
                }
                if (response.statusCode == "TooManyRequests")
                {
                    for (int s = 0; s <= response.retryInSeconds; s += 15)
                    {
                        Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
                        Thread.Sleep(15000);
                    }
                    return response.statusCode;
                }

                // Pagination loop
                while (true)
                {
                    int noteCount = response?.response?.notes?.Count ?? 0;
                    Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");

                    if (response?.response == null)
                    {
                        Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
                        return "FAILURE";
                    }
                    if (response.response.notes == null)
                    {
                        Console.WriteLine("##### Notes Null - WHY? ###");
                        return "FAILURE";
                    }

                    // Accumulate notes
                    allNotes.AddRange(response.response.notes);

                    // Pagination: check for next
                    var nextLink = response.response._links?.next;
                    if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
                    {
                        Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
                        break;
                    }

                    beforeTimestamp = nextLink.query_params.before_timestamp;
                    page++;
                    if (page > maxPages)
                    {
                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination.");
                        break;
                    }

                    // Fetch next page
                    response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
                }

                Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");

                // Process all accumulated notes
                foreach (var note in allNotes)
                {
                    note.reblog_parent_blog_name = post.Item1;
                    note.post_id = post.Item2.ToString();
                    if (note.type == "reply")
                        hasReplies = true;
                    DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
                }

                return "Success";
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
            return "UNKNOWN";
        }

        static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
        {
            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);

            RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
            {
                PermitLimit = 300,
                QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                QueueLimit = 1,
                Window = TimeSpan.FromMinutes(1),
                SegmentsPerWindow = 60,
                AutoReplenishment = true
            });

            try
            {
                using (StreamWriter sw = new StreamWriter(outPath, true))
                {
                    while (posts.Count > 0)
                    {
                        var post = posts[0]; // Process the first post in the list
                        string status;

                        using RateLimitLease lease = limiter.AttemptAcquire(1);
                        if (lease.IsAcquired)
                        {
                            Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
                            status = await GrabNotes(post);
                        }
                        else
                        {
                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                            return;
                        }

                        if (status == "Success")
                        {
                            DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
                        }
                        else
                        {
                            Console.WriteLine("GrabNotes Result: " + status);
                        }

                        // Re-fetch the updated list after processing the current post
                        posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }


        static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "")
        {
            // Get all directories in the current directory and sort them alphabetically
            var directories = Directory.GetDirectories(path);
            Array.Sort(directories, StringComparer.InvariantCulture);

            foreach (var directory in directories)
            {
                Console.WriteLine("Directory: " + directory);
                TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories
            }

            try
            {
                bool headerWasWritten = false;
                // Process all files in the current directory
                foreach (var file in Directory.GetFiles(path))
                {
                    if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
                    {
                        try
                        {
                            var urls = new List<string>();
                            var reblog = new ReblogRecord();

                            foreach (string line in File.ReadLines(file))
                            {
                                if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
                                {
                                    if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
                                    {
                                        if (!reblog.reblogURL.Contains("deactivated")
                                            && reblog.reblogURL.Length != 0
                                            && ContainsAny(reblog.reblogURL, contains))
                                        {
                                            DirectoryInfo currentDir = new DirectoryInfo(path);
                                            if (!headerWasWritten)
                                            {
                                                headerWasWritten = true;
                                            }
                                            string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
                                            urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
                                            DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                reblog.title, false);
                                            postsAdded++;

                                            // Output hyperlink and post date
                                            //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                        }
                                    }
                                    reblog = new ReblogRecord();
                                    reblog.postID = line.Substring(9).Trim();
                                }
                                if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
                                {
                                    //reblog = new ReblogRecord();

                                    reblog.reblogURL = line.Substring(12).Trim();
                                }
                                if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.reblogName = line.Substring(13).Trim();
                                }
                                if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.downloadedFiles = line.Substring(17).Trim();
                                }
                                if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.reblogKey = line.Substring(11).Trim();
                                }
                                if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.date = line.Substring(6).Trim();
                                }
                                if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.body = line.Substring(6).Trim();
                                }
                                if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.postURL = line.Substring(10).Trim();
                                }
                                if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.answer = line.Substring(8).Trim();
                                }
                                if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.audioCaption = line.Substring(15).Trim();
                                }
                                if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.blogName = line.Substring(11).Trim();
                                }
                                if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.link = line.Substring(6).Trim();
                                }
                                if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.photoCaption = line.Substring(15).Trim();
                                }
                                if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.photoURL = line.Substring(11).Trim();
                                }
                                if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.question = line.Substring(10).Trim();
                                }
                                if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.quote = line.Substring(7).Trim();
                                }
                                if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.slug = line.Substring(6).Trim();
                                }
                                if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.summary = line.Substring(9).Trim();
                                }
                                if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.tags = line.Substring(6).Trim();
                                }
                                if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
                                {
                                    reblog.title = line.Substring(7).Trim();
                                }

                                if ((reblog.downloadedFiles != "."
                                        || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
                                        || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
                                {
                                    if ((ContainsAny(reblog.downloadedFiles, contains)
                                        || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
                                        || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
                                        //&& !reblog.reblogURL.Contains("deactivated")
                                        || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
                                    {
                                        DirectoryInfo currentDir = new DirectoryInfo(path);
                                        if (!headerWasWritten)
                                        {
                                            headerWasWritten = true;
                                        }
                                        //if (reblog.reblogURL.Contains("/blog/private")
                                        //    || reblog.body.Contains("/blog/private"))
                                        //{
                                        //    Console.WriteLine(reblog.postID);
                                        //    Console.WriteLine(reblog.postURL);
                                        //    Console.WriteLine(reblog.date);
                                        //    Console.WriteLine(reblog.body);
                                        //    Console.WriteLine(reblog.reblogKey);
                                        //    Console.WriteLine(reblog.reblogURL);
                                        //    Console.WriteLine(reblog.reblogName);
                                        //    Console.WriteLine(reblog.downloadedFiles);
                                        //}
                                        string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
                                        urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
                                        DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                reblog.title, true);
                                        postsAdded++;

                                        // Output hyperlink and post date
                                        //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                    }
                                }
                            }
                            urls.Sort();


                            //using (StreamWriter sw = new StreamWriter(outPath, true))
                            //{
                            //    foreach (string line in urls.Distinct())
                            //    {
                            //        sw.WriteLine(line);
                            //    }
                            //}
                        }
                        catch (Exception e)
                        {
                            Console.WriteLine("The file could not be read:");
                            Console.WriteLine(e.Message);
                        }
                    }
                    // Add your file processing logic here
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"An error occurred: {ex.Message}");
            }
        }


        static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
        {
            var directories = Directory.GetDirectories(path);
            Array.Sort(directories, StringComparer.InvariantCulture);

            foreach (var directory in directories)
            {
                //Console.WriteLine("Directory: " + directory);
                TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
            }
            try
            {
                bool headerWasWritten = false;
                foreach (var file in Directory.GetFiles(path))
                {
                    if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
                    {
                        try
                        {
                            var urls = new List<string>();
                            var reblog = new ReblogRecord();

                            foreach (string line in File.ReadLines(file))
                            {
                                if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
                                {
                                    Console.WriteLine(file);
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            Console.WriteLine("The file could not be read:");
                            Console.WriteLine(e.Message);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"An error occurred: {ex.Message}");
            }
        }
    }
}