using System.Threading.RateLimiting; using Microsoft.Extensions.Http.Resilience; using Microsoft.Extensions; using Microsoft.Extensions.Configuration; using System.Configuration; namespace URLNotesGrabberCORE { internal class Program { static void Main(string[] args) { IConfiguration config = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false) .AddCommandLine(args) .Build(); var settings = config.GetSection("appSettings"); if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB { TraverseDirectory( settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs")); } else switch(args[0]) { case "-n": //Manual test var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult(); List> notes = new List>(); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } while (response.response._links != null) { response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } } break; case "-p": //write post's blogs to file WritePostBlogsToFile(settings.GetValue("PathOutputPosts")); break; case "-b": //write blogs to file WriteBlogsToFile(settings.GetValue("PathOutputBlogs")); break; case "-pn": //collect notes from all posts CollectNotes(settings.GetValue("PathOutput")); break; case "-br": //collect notes from all posts WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), true); break; } System.Console.WriteLine(":/"); System.Console.ReadKey(); } static void WritePostBlogsToFile(string outPath) { List> posts = DataAccess.GetPosts(); using (StreamWriter sw = new StreamWriter(outPath, true)) { List blogs = new List(); foreach (var post in posts) { blogs.Add(post.Item1); } blogs = blogs.Distinct().ToList(); blogs.Sort(); foreach (var blog in blogs) { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); } } } static void WriteBlogsToFile(string outPath, bool reblogsOnly = false) { List blogs = DataAccess.GetBlogs(reblogsOnly); blogs.Sort(); using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (var blog in blogs) { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); DataAccess.UpdatePostBlogOutput(blog); } } } protected static bool ContainsAny(string input) { if (string.IsNullOrEmpty(input)) return false; var items = new List(); items.Add("zombaee"); items.Add("zomb-eh"); items.Add("ahzombae"); items.Add("thebugandme"); items.Add("lovingbabybug"); items.Add("swarthyvillain"); foreach (string item in items) { if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0) { return true; } } return false; } static async Task GrabNotes(Tuple post) { try { int APICount = DataAccess.GetAPICount(); Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now); var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult(); List> notes = new List>(); if (response.statusCode == "NotFound") { Thread.Sleep(1000); DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); return response.statusCode; } if (response.statusCode == "TooManyRequests") { for( int s = 0; s <= response.retryInSeconds; s++) { Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s); Thread.Sleep(1000); } return response.statusCode; } if (response.response != null) { Console.WriteLine("Notes\t" + response.response.notes.Count); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } } if (response.response == null) return "FAILURE"; while (response.response != null && response.response._links != null) { response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); if (response.response.notes is null) return "NULL NOTES"; Console.WriteLine("Notes\t" + response.response.notes.Count); foreach (var note in response.response.notes) { note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); //notes.Add(new Tuple(note.blog_url, note.type)); if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) break; } } return "Success"; } catch(Exception ex) { Console.WriteLine(ex.ToString()); } return "UNKNOWN"; } static async void CollectNotes(string outPath) { List> posts = DataAccess.GetPosts(true); RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { PermitLimit = 300, QueueProcessingOrder = QueueProcessingOrder.OldestFirst, QueueLimit = 1, Window = TimeSpan.FromMinutes(1), SegmentsPerWindow = 60, AutoReplenishment = true} ); try { using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (var post in posts) { string status; using RateLimitLease lease = limiter.AttemptAcquire(1); if (lease.IsAcquired) { status = await GrabNotes(post); } else { Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); return; } if(status == "Success") DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); else Console.WriteLine("GrabNotes Result: " + status); } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } static void TraverseDirectory(string path, string outPath) { // Get all directories in the current directory and sort them alphabetically var directories = Directory.GetDirectories(path); Array.Sort(directories, StringComparer.InvariantCulture); using (StreamWriter sw = new StreamWriter(outPath, true)) { //sw.WriteLine("=====" + path); } foreach (var directory in directories) { Console.WriteLine("Directory: " + directory); TraverseDirectory(directory, outPath); // Recursively traverse subdirectories } try { bool headerWasWritten = false; // Process all files in the current directory foreach (var file in Directory.GetFiles(path)) { if (file.EndsWith(".txt")) { Console.WriteLine($"=====File: {file}"); using (StreamWriter sw = new StreamWriter(outPath, true)) { //sw.WriteLine("---" + file); } try { var urls = new List(); var reblog = new ReblogRecord(); foreach (string line in File.ReadLines(file)) { //if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated")) if (line.StartsWith(@"Post id:")) { reblog = new ReblogRecord(); reblog.postID = line.Substring(9).Trim(); } if (line.StartsWith(@"Reblog url:")) { //reblog = new ReblogRecord(); reblog.reblogURL = line.Substring(12).Trim(); } if (line.StartsWith(@"Reblog name:")) { reblog.reblogName = line.Substring(13).Trim(); } if (line.StartsWith(@"Downloaded files:")) { reblog.downloadedFiles = line.Substring(17).Trim(); } if (line.StartsWith(@"Reblog key:")) { reblog.reblogKey = line.Substring(11).Trim(); } if (line.StartsWith(@"Date:")) { reblog.date = line.Substring(6).Trim(); } if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") { if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated")) { DirectoryInfo currentDir = new DirectoryInfo(path); if (!headerWasWritten) { headerWasWritten = true; } Console.WriteLine(reblog.postID); Console.WriteLine(reblog.date); Console.WriteLine(reblog.reblogKey); Console.WriteLine(reblog.reblogURL); Console.WriteLine(reblog.reblogName); Console.WriteLine(reblog.downloadedFiles); string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL ); } } } urls.Sort(); using (StreamWriter sw = new StreamWriter(outPath, true)) { foreach (string line in urls.Distinct()) { sw.WriteLine(line); } } } catch (Exception e) { Console.WriteLine("The file could not be read:"); Console.WriteLine(e.Message); } } // Add your file processing logic here } } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); } } } }