using System.Threading.RateLimiting; using Microsoft.Extensions.Http.Resilience; using Microsoft.Extensions; using Microsoft.Extensions.Configuration; using System.Configuration; using System.Threading; using Microsoft.Extensions.Diagnostics.Latency; using static System.Runtime.InteropServices.JavaScript.JSType; namespace URLNotesGrabberCORE { internal class Program { static void Main(string[] args) { IConfiguration config = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false) .AddCommandLine(args) .Build(); var settings = config.GetSection("appSettings"); // Setup Dual Logging string logPath = "console_output.log"; StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true }; DualLogger dualLogger = new DualLogger(Console.Out, fileWriter); Console.SetOut(dualLogger); List contains = settings.GetValue("ContainsList").Split(',').ToList(); if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB { TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains); } else { switch (args[0]) { case "-?": Console.WriteLine("\t Parse .txt files to find blogs"); Console.WriteLine("-?\t Usage help"); Console.WriteLine("-parse\t Parse .txt files with specified blogname"); Console.WriteLine("-test\t Calls API for given blogname and postID"); Console.WriteLine("-posts\t For each Post in DB, write blogname to file"); Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file"); Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes"); Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file "); Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range "); Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file "); break; case "-parse": string blogNameToParse = args[1]; TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, blogNameToParse); break; case "-test": #region Manual test string blogName = args[1]; long postID = long.Parse(args[2]); var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult(); List> notes = new List>(); // Check for API errors if (response.statusCode == "NotFound") { Console.WriteLine($"ERROR: Post not found - {blogName}/{postID}"); break; } if (response.statusCode == "TooManyRequests") { Console.WriteLine($"ERROR: Rate limited - retry in {response.retryInSeconds} seconds"); break; } if (!string.IsNullOrEmpty(response.statusCode)) { Console.WriteLine($"ERROR: API returned status code: {response.statusCode}"); break; } // Check for null response if (response.response == null) { Console.WriteLine("ERROR: API response is null"); break; } // Check for null notes if (response.response.notes == null) { Console.WriteLine("ERROR: Notes collection is null (post may have 0 notes)"); break; } Console.WriteLine($"Processing {response.response.notes.Count} notes..."); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } while (response.response._links != null) { response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); if (response.response?.notes == null) { Console.WriteLine("ERROR: Notes collection became null during pagination"); break; } Console.WriteLine($"Processing {response.response.notes.Count} more notes..."); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ; //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } } Console.WriteLine("Test completed successfully"); #endregion break; case "-post": TraverseDirectoryForCorruption(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains); break; case "-posts": //write post's blogs to file WritePostBlogsToFile(settings.GetValue("PathOutputPosts")); break; case "-blogs": //write blogs to file WriteBlogsToFile(settings.GetValue("PathOutputBlogs")); break; case "-collect": //collect notes from all posts bool withoutNotesOnly = true; if(args.Length != 2) { Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--"); break; } if (args.Length > 1 && args[1] is not null) { if (args[1] == "1") { withoutNotesOnly = true; Console.WriteLine("Parsed"); } else { withoutNotesOnly = false; Console.WriteLine("--NOT Parsed"); } Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]); } CollectNotes(settings.GetValue("PathOutput"), withoutNotesOnly); break; case "-blogsR": //collect notes from all posts WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), true); break; case "-blogsO": //collect notes from all posts int from = 1, to = 999999, top = 100; if (args[1] is not null && args[2] is not null && args[3] is not null) { from = int.Parse(args[1]); to = int.Parse(args[2]); top = int.Parse(args[3]); } else { Console.WriteLine("--Expected FROM TO--"); } WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), false, from, to, top); break; case "-bop": //collect notes from all posts from = 1; to = 999999; top = 100; if (args[1] is not null && args[2] is not null && args[3] is not null) { from = int.Parse(args[1]); to = int.Parse(args[2]); top = int.Parse(args[3]); } else { Console.WriteLine("--Expected FROM TO--"); } WriteBlogsToFileAll(settings.GetValue("PathOutputBlogs"), false, from, to, top); break; case "-replies": //colection posts with replies try { System.IO.File.Delete(settings.GetValue("PathOutputReplies")); } catch { } WriteRepliesToFile(settings.GetValue("PathOutputBlogs"), false); break; default: Console.WriteLine("** Unknown Command ** " + args[0]); break; } } System.Console.WriteLine(":/"); //System.Console.ReadKey(); } static void WritePostBlogsToFile(string outPath) { List> posts = DataAccess.GetPosts(); using (StreamWriter sw = new StreamWriter(outPath, true)) { List blogs = new List(); foreach (var post in posts) { blogs.Add(post.Item1); } blogs = blogs.Distinct().ToList(); blogs.Sort(); blogs.Reverse(); foreach (var blog in blogs) { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); } } } static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100) { List blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top); blogs.Sort(); blogs.Reverse(); using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (var blog in blogs) { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); DataAccess.UpdateBlogOutput(blog); } } } static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100) { List blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top); blogs.Sort(); blogs.Reverse(); using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (var blog in blogs) { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); DataAccess.UpdateBlogOutput(blog); } } } static void WriteRepliesToFile(string outPath, bool reblogsOnly = false) { List> posts = DataAccess.GetReplies(); using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (var post in posts) { Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2); sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2); } } } protected static bool ContainsAny(string input, List contains) { if (string.IsNullOrEmpty(input)) return false; foreach (string item in contains) { if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0) { return true; } } return false; } static async Task GrabNotes(Tuple post) { try { int APICount = DataAccess.GetAPICount(); Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount); var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult(); List> notes = new List>(); if (response.statusCode == "NotFound") { Thread.Sleep(1000); DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); return response.statusCode; } if (response.statusCode == "TooManyRequests") { for (int s = 0; s <= response.retryInSeconds; s += 15) { Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString()); Thread.Sleep(15000); } return response.statusCode; } if (response.response != null && response.response.notes != null) { Console.WriteLine("Notes\t" + response.response.notes.Count); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } } else if (response.response == null) { Console.WriteLine("##### Response Null - API Failure? ###"); return "FAILURE"; } else if (response.response.notes == null) { Console.WriteLine("##### Notes Null - WHY? ###"); return "FAILURE"; } else { if (response.response == null) { Console.WriteLine("response.response == null"); return "FAILURE"; } while (response.response != null && response.response._links != null && long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3) { response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); if (response.response.notes is null) { Console.WriteLine("response.response.notes == null"); return "NULL NOTES"; } Console.WriteLine("Notes\t" + response.response.notes.Count); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } } } return "Success"; } catch(Exception ex) { Console.WriteLine(ex.ToString()); } return "UNKNOWN"; } static async void CollectNotes(string outPath, bool withoutNotesOnly = true) { List> posts = DataAccess.GetPosts(withoutNotesOnly); RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { PermitLimit = 300, QueueProcessingOrder = QueueProcessingOrder.OldestFirst, QueueLimit = 1, Window = TimeSpan.FromMinutes(1), SegmentsPerWindow = 60, AutoReplenishment = true} ); try { using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (var post in posts) { string status; using RateLimitLease lease = limiter.AttemptAcquire(1); if (lease.IsAcquired) { Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString()); Thread.Sleep(1000); status = await GrabNotes(post); } else { Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); return; } if(status == "Success") DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); else Console.WriteLine("GrabNotes Result: " + status); } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } static void TraverseDirectory(string path, string outPath, List contains, string blogName = "") { // Get all directories in the current directory and sort them alphabetically var directories = Directory.GetDirectories(path); Array.Sort(directories, StringComparer.InvariantCulture); //using (StreamWriter sw = new StreamWriter(outPath, true)) //{ //sw.WriteLine("=====" + path); //} foreach (var directory in directories) { Console.WriteLine("Directory: " + directory); TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories } try { bool headerWasWritten = false; // Process all files in the current directory foreach (var file in Directory.GetFiles(path)) { if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == "")) { //Console.WriteLine($"=====File: {file}"); //using (StreamWriter sw = new StreamWriter(outPath, true)) //{ //sw.WriteLine("---" + file); //} try { var urls = new List(); var reblog = new ReblogRecord(); foreach (string line in File.ReadLines(file)) { //if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated")) if (line.StartsWith(@"Post id:")) { // if there were no files, let's still collect post info if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") { if (!reblog.reblogURL.Contains(@"deactivated") && reblog.reblogURL.Length != 0 && ContainsAny(reblog.reblogURL, contains)) { DirectoryInfo currentDir = new DirectoryInfo(path); if (!headerWasWritten) { headerWasWritten = true; } //if (reblog.reblogURL.Contains("/blog/private") // || reblog.body.Contains("/blog/private")) //{ // Console.WriteLine(reblog.postID); // Console.WriteLine(reblog.postURL); // Console.WriteLine(reblog.date); // Console.WriteLine(reblog.body); // Console.WriteLine(reblog.reblogKey); // Console.WriteLine(reblog.reblogURL); // Console.WriteLine(reblog.reblogName); // Console.WriteLine(reblog.downloadedFiles); //} string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, reblog.title, false); } } reblog = new ReblogRecord(); reblog.postID = line.Substring(9).Trim(); } if (line.StartsWith(@"Reblog url:")) { //reblog = new ReblogRecord(); reblog.reblogURL = line.Substring(12).Trim(); } if (line.StartsWith(@"Reblog name:")) { reblog.reblogName = line.Substring(13).Trim(); } if (line.StartsWith(@"Downloaded files:")) { reblog.downloadedFiles = line.Substring(17).Trim(); } if (line.StartsWith(@"Reblog key:")) { reblog.reblogKey = line.Substring(11).Trim(); } if (line.StartsWith(@"Date:")) { reblog.date = line.Substring(6).Trim(); } if (line.StartsWith(@"Body:")) { reblog.body = line.Substring(6).Trim(); } if (line.StartsWith(@"Post url:")) { reblog.postURL = line.Substring(10).Trim(); } if (line.StartsWith(@"Answer:")) { reblog.answer = line.Substring(8).Trim(); } if (line.StartsWith(@"Audio Caption:")) { reblog.audioCaption = line.Substring(15).Trim(); } if (line.StartsWith(@"Blog Name:")) { reblog.blogName = line.Substring(11).Trim(); } if (line.StartsWith(@"Link:")) { reblog.link = line.Substring(6).Trim(); } if (line.StartsWith(@"Photo Caption:")) { reblog.photoCaption = line.Substring(15).Trim(); } if (line.StartsWith(@"Photo url:")) { reblog.photoURL = line.Substring(11).Trim(); } if (line.StartsWith(@"Question:")) { reblog.question = line.Substring(10).Trim(); } if (line.StartsWith(@"Quote:")) { reblog.quote = line.Substring(7).Trim(); } if (line.StartsWith(@"Slug:")) { reblog.slug = line.Substring(6).Trim(); } if (line.StartsWith(@"Summary:")) { reblog.summary = line.Substring(9).Trim(); } if (line.StartsWith(@"Tags:")) { reblog.tags = line.Substring(6).Trim(); } if (line.StartsWith(@"Title:")) { reblog.title = line.Substring(7).Trim(); } if ((reblog.downloadedFiles != "." || reblog.reblogURL.Contains("/blog/private") || reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") { if ((ContainsAny(reblog.downloadedFiles, contains) || reblog.reblogURL.Contains("/blog/private") || reblog.body.Contains("/blog/private")) // && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated")) { DirectoryInfo currentDir = new DirectoryInfo(path); if (!headerWasWritten) { headerWasWritten = true; } //if (reblog.reblogURL.Contains("/blog/private") // || reblog.body.Contains("/blog/private")) //{ // Console.WriteLine(reblog.postID); // Console.WriteLine(reblog.postURL); // Console.WriteLine(reblog.date); // Console.WriteLine(reblog.body); // Console.WriteLine(reblog.reblogKey); // Console.WriteLine(reblog.reblogURL); // Console.WriteLine(reblog.reblogName); // Console.WriteLine(reblog.downloadedFiles); //} string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, reblog.title, true); } } } urls.Sort(); //using (StreamWriter sw = new StreamWriter(outPath, true)) //{ // foreach (string line in urls.Distinct()) // { // sw.WriteLine(line); // } //} } catch (Exception e) { Console.WriteLine("The file could not be read:"); Console.WriteLine(e.Message); } } // Add your file processing logic here } } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); } } static void TraverseDirectoryForCorruption(string path, string outPath, List contains, string blogName = "") { var directories = Directory.GetDirectories(path); Array.Sort(directories, StringComparer.InvariantCulture); foreach (var directory in directories) { //Console.WriteLine("Directory: " + directory); TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories } try { bool headerWasWritten = false; foreach (var file in Directory.GetFiles(path)) { if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == "")) { try { var urls = new List(); var reblog = new ReblogRecord(); foreach (string line in File.ReadLines(file)) { if (!line.StartsWith(@"Post id:") && line.Contains(@"id:")) { Console.WriteLine(file); } } } catch (Exception e) { Console.WriteLine("The file could not be read:"); Console.WriteLine(e.Message); } } } } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); } } } }