From 225934d3b9dda7423ea80287db587161dcf18499 Mon Sep 17 00:00:00 2001 From: JB Date: Wed, 30 Oct 2024 09:14:53 -0500 Subject: [PATCH] Add project files. --- URLNotesGrabberCORE.sln | 25 ++ URLNotesGrabberCORE/DataAccess.cs | 405 ++++++++++++++++++ URLNotesGrabberCORE/Program.cs | 345 +++++++++++++++ .../Properties/launchSettings.json | 8 + URLNotesGrabberCORE/ResponseNotes.cs | 81 ++++ URLNotesGrabberCORE/TL.db | Bin 0 -> 28672 bytes .../URLNotesGrabberCORE.csproj | 34 ++ URLNotesGrabberCORE/appsettings.json | 10 + 8 files changed, 908 insertions(+) create mode 100644 URLNotesGrabberCORE.sln create mode 100644 URLNotesGrabberCORE/DataAccess.cs create mode 100644 URLNotesGrabberCORE/Program.cs create mode 100644 URLNotesGrabberCORE/Properties/launchSettings.json create mode 100644 URLNotesGrabberCORE/ResponseNotes.cs create mode 100644 URLNotesGrabberCORE/TL.db create mode 100644 URLNotesGrabberCORE/URLNotesGrabberCORE.csproj create mode 100644 URLNotesGrabberCORE/appsettings.json diff --git a/URLNotesGrabberCORE.sln b/URLNotesGrabberCORE.sln new file mode 100644 index 0000000..1daca98 --- /dev/null +++ b/URLNotesGrabberCORE.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.9.34902.65 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "URLNotesGrabberCORE", "URLNotesGrabberCORE\URLNotesGrabberCORE.csproj", "{D56E1CA5-02C8-47CF-B6C8-8952CBAAA1B0}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D56E1CA5-02C8-47CF-B6C8-8952CBAAA1B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D56E1CA5-02C8-47CF-B6C8-8952CBAAA1B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D56E1CA5-02C8-47CF-B6C8-8952CBAAA1B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D56E1CA5-02C8-47CF-B6C8-8952CBAAA1B0}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9F352AFC-1F2F-4108-833F-8669AF618481} + EndGlobalSection +EndGlobal diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs new file mode 100644 index 0000000..c0c4e67 --- /dev/null +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -0,0 +1,405 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Data.SQLite; +using RestSharp.Authenticators.OAuth; +using RestSharp.Authenticators; +using RestSharp; +using Newtonsoft.Json; + +namespace URLNotesGrabberCORE +{ + + class ReblogRecord + { + public string postID; + public string reblogURL; + public string reblogName; + public string downloadedFiles; + public string reblogKey; + public string date; + + public ReblogRecord() + { + postID = "."; + reblogURL = "."; + reblogName = "."; + downloadedFiles = "."; + reblogKey = "."; + date = "."; + } + } + + internal class DataAccess + { + #region Adds + public static void AddBlog(string blogName, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + if (blogName.Contains("deact")) + return; + + try + { + connection.Open(); + + string sql = "INSERT INTO Blogs (BlogName) values('" + blogName + "')"; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + + public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db") + { + try { AddBlog(blogName, DBPath); } catch { } + + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')"; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + + public static void AddAPICount(string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "INSERT INTO DailyAPICount (Date) values('" + DateTime.Today.ToShortDateString() + "')"; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + //Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + + public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db") + { + try { AddPost(rootBlogName, postID, DBPath); } catch { } + try { AddBlog(noteBlogName, DBPath); } catch { } + + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values('" + rootBlogName + "', '" + noteBlogName + "', " + postID + ", " + timestamp + ", '" + type + "')"; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") + { + Console.WriteLine(ex.Message); + Console.WriteLine("^^^^^ - SHORTCUT"); + return true; + } + } + finally + { + connection.Close(); + } + return false; + } + #endregion Adds + + #region Gets + public static List> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + List> posts = new List>(); + + try + { + connection.Open(); + + string sql = "SELECT " + + " MIN(blogName) as blogName, " + + " postID " + + "FROM" + + " Posts "; + + if (withoutNotesOnly) + sql += " WHERE HasNotesGathered = 0"; + + sql += " GROUP BY postID ORDER BY reblogURL, PostID"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + Tuple post = default; + string blog = null; + long id = 0; + id = reader.GetInt64(1); // Assuming Id is the first column + blog = reader.GetString(0); // Assuming Title is the second column + + post = new Tuple(blog, id); + posts.Add(post); + } + } + } + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + return posts; + } + + + public static int GetAPICount(string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + int count = 0; + + try { AddAPICount(); } catch { } + + try + { + connection.Open(); + + string sql = "SELECT " + + " APICount " + + "FROM " + + " DailyAPICount " + + " WHERE [Date] = '" + + DateTime.Today.ToShortDateString() + "'"; + + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + count = reader.GetInt32(0); // Assuming Id is the first column + } + } + } + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + return count; + } + + + public static List GetBlogs(bool reblogsOnly, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + List blogs = new List(); + + try + { + connection.Open(); + string sql = ""; + if (reblogsOnly) + sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' ORDER BY BlogName"; + else + sql = "SELECT * FROM Blogs ORDER BY BlogName"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + string blog = null; + blog = reader.GetString(0); // Assuming Title is the second column + + blogs.Add(blog); + } + } + } + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + return blogs; + } + #endregion Gets + + #region Updates + + + public static void UpdatePostMarkNotesCollected(string blogName, long postID, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + + public static int UpdateAPICount(string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + int APICount = DataAccess.GetAPICount(); + APICount++; + + try + { + connection.Open(); + + string sql = "UPDATE DailyAPICount SET APICount = " + APICount + + " WHERE [Date] = '" + + DateTime.Today.ToShortDateString() + "'"; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + + return APICount; + } + #endregion Updates + } + + internal class APIAccess + { + public static string Blog { get; set; } + + const string CONSUMER_KEY = "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3"; + const string CONSUMER_SECRET = "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG"; + const string OAUTH_TOKEN = "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J"; + const string OAUTH_TOKEN_SECRET = "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w"; + + public static async Task GrabNotes(string blog, long ID, string timestamp = null) + { + var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]"; + URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString()); + if (timestamp != null) + { + URL += "&before_timestamp=" + timestamp; + Thread.Sleep(500); + } + var client = new RestClient(URL); + var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY, + consumerSecret: CONSUMER_SECRET, + token: OAUTH_TOKEN, + tokenSecret: OAUTH_TOKEN_SECRET, + OAuthSignatureMethod.HmacSha1 + ); + //oAuth1.Realm = + + client.Authenticator = oAuth1; + + var request = new RestRequest(URL, Method.Get); + + var response = client.Execute(request); + + var myJsonResponse = response.Content; + Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount()); + Root myDeserializedClass = new Root(); + + try + { + myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); + + if (myDeserializedClass.response.total_notes != 0) + { + DataAccess.AddBlog(blog); + //DataAccess.AddPost(blog, ID); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + + if (!response.IsSuccessful) + { + Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription); + myDeserializedClass.statusCode = response.StatusCode.ToString(); + } + } + + return myDeserializedClass; + } + } +} diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs new file mode 100644 index 0000000..dd4daec --- /dev/null +++ b/URLNotesGrabberCORE/Program.cs @@ -0,0 +1,345 @@ +using System.Threading.RateLimiting; +using Microsoft.Extensions.Http.Resilience; +using Microsoft.Extensions; +using Microsoft.Extensions.Configuration; +using System.Configuration; + +namespace URLNotesGrabberCORE +{ + internal class Program + { + + static void Main(string[] args) + { + IConfiguration config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false) + .AddCommandLine(args) + .Build(); + var settings = config.GetSection("appSettings"); + + if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB + { + TraverseDirectory( settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs")); + } + else if (args[0] == "-n") //Manual test + { + /*var response = */ + //var url = await CreateProductAsync(product); + //var t = await GrabNotes(); + var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult(); + List> notes = new List>(); + + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + + while (response.response._links != null) + { + response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); + + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + } + //var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList(); + //notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList(); + + //using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true)) + //{ + // foreach (var note in notes) + // { + // using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true)) + // } + //} + + } + else if (args[0] == "-p") //write post's blogs to file + { + WritePostBlogsToFile(settings.GetValue("PathOutputPosts")); + } + else if (args[0] == "-b") //write blogs to file + { + WriteBlogsToFile(settings.GetValue("PathOutputBlogs")); + } + else if (args[0] == "-pn") //collect notes from all posts + { + CollectNotes(settings.GetValue("PathOutput")); + } + else if (args[0] == "-br") //collect notes from all posts + { + WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), true); + } + + + + System.Console.WriteLine(":/"); + System.Console.ReadKey(); + } + + static void WritePostBlogsToFile(string outPath) + { + List> posts = DataAccess.GetPosts(); + + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + List blogs = new List(); + foreach (var post in posts) + { + blogs.Add(post.Item1); + } + blogs = blogs.Distinct().ToList(); + blogs.Sort(); + + foreach (var blog in blogs) + { + Console.WriteLine(blog); + sw.WriteLine(blog + ".tumblr.com"); + } + } + } + + static void WriteBlogsToFile(string outPath, bool reblogsOnly = false) + { + List blogs = DataAccess.GetBlogs(reblogsOnly); + blogs.Sort(); + + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + foreach (var blog in blogs) + { + Console.WriteLine(blog); + sw.WriteLine(blog + ".tumblr.com"); + } + } + } + + protected static bool ContainsAny(string input) + { + if (string.IsNullOrEmpty(input)) return false; + var items = new List(); + items.Add("zombaee"); + items.Add("zomb-eh"); + items.Add("ahzombae"); + items.Add("thebugandme"); + items.Add("lovingbabybug"); + items.Add("swarthyvillain"); + + + foreach (string item in items) + { + if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + return false; + } + + static async Task GrabNotes(Tuple post) + { + try + { + int APICount = DataAccess.GetAPICount(); + + Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now); + var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult(); + List> notes = new List>(); + + if (response.statusCode == "NotFound") + return; + if (response.statusCode == "TooManyRequests") + return; + + if (response.response != null) + { + Console.WriteLine("Notes\t" + response.response.notes.Count); + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + } + if (response.response == null) + throw new Exception("Response is null"); + + while (response.response != null && response.response._links != null) + { + response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); + + Console.WriteLine("Notes\t" + response.response.notes.Count); + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + } + } + catch(Exception ex) { Console.WriteLine(ex.ToString()); } + } + + static async void CollectNotes(string outPath) + { + List> posts = DataAccess.GetPosts(true); + + RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { + PermitLimit = 300, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + QueueLimit = 1, + Window = TimeSpan.FromMinutes(1), + SegmentsPerWindow = 60, + AutoReplenishment = true} ); + + try + { + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + foreach (var post in posts) + { + using RateLimitLease lease = limiter.AttemptAcquire(1); + if (lease.IsAcquired) + { + await GrabNotes(post); + } + else + { + Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); + return; + } + DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + } + + + static void TraverseDirectory(string path, string outPath) + { + // Get all directories in the current directory and sort them alphabetically + var directories = Directory.GetDirectories(path); + Array.Sort(directories, StringComparer.InvariantCulture); + + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + //sw.WriteLine("=====" + path); + } + foreach (var directory in directories) + { + Console.WriteLine("Directory: " + directory); + TraverseDirectory(directory, outPath); // Recursively traverse subdirectories + } + try + { + bool headerWasWritten = false; + // Process all files in the current directory + foreach (var file in Directory.GetFiles(path)) + { + if (file.EndsWith(".txt")) + { + Console.WriteLine($"=====File: {file}"); + + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + //sw.WriteLine("---" + file); + } + + try + { + var urls = new List(); + var reblog = new ReblogRecord(); + + foreach (string line in File.ReadLines(file)) + { + //if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated")) + if (line.StartsWith(@"Post id:")) + { + reblog = new ReblogRecord(); + reblog.postID = line.Substring(9).Trim(); + } + if (line.StartsWith(@"Reblog url:")) + { + //reblog = new ReblogRecord(); + + reblog.reblogURL = line.Substring(12).Trim(); + } + if (line.StartsWith(@"Reblog name:")) + { + reblog.reblogName = line.Substring(13).Trim(); + } + if (line.StartsWith(@"Downloaded files:")) + { + reblog.downloadedFiles = line.Substring(17).Trim(); + } + if (line.StartsWith(@"Reblog key:")) + { + reblog.reblogKey = line.Substring(11).Trim(); + } + if (line.StartsWith(@"Date:")) + { + reblog.date = line.Substring(6).Trim(); + } + + if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") + { + if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated")) + { + DirectoryInfo currentDir = new DirectoryInfo(path); + if (!headerWasWritten) + { + headerWasWritten = true; + } + Console.WriteLine(reblog.postID); + Console.WriteLine(reblog.date); + Console.WriteLine(reblog.reblogKey); + Console.WriteLine(reblog.reblogURL); + Console.WriteLine(reblog.reblogName); + Console.WriteLine(reblog.downloadedFiles); + string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); + urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); + DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL ); + } + } + } + urls.Sort(); + + + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + foreach (string line in urls.Distinct()) + { + sw.WriteLine(line); + } + } + } + catch (Exception e) + { + Console.WriteLine("The file could not be read:"); + Console.WriteLine(e.Message); + } + } + // Add your file processing logic here + } + } + catch (Exception ex) + { + Console.WriteLine($"An error occurred: {ex.Message}"); + } + } + } +} diff --git a/URLNotesGrabberCORE/Properties/launchSettings.json b/URLNotesGrabberCORE/Properties/launchSettings.json new file mode 100644 index 0000000..1e999bd --- /dev/null +++ b/URLNotesGrabberCORE/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "URLNotesGrabberCORE": { + "commandName": "Project", + "commandLineArgs": "-br" + } + } +} \ No newline at end of file diff --git a/URLNotesGrabberCORE/ResponseNotes.cs b/URLNotesGrabberCORE/ResponseNotes.cs new file mode 100644 index 0000000..f3315e8 --- /dev/null +++ b/URLNotesGrabberCORE/ResponseNotes.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; + + +namespace URLNotesGrabberCORE +{ + internal class ResponseNotes + { + } + // Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); + public class AvatarUrl + { + [JsonProperty("64")] + public string _64 { get; set; } + + [JsonProperty("128")] + public string _128 { get; set; } + } + + public class Links + { + public Next next { get; set; } + } + + public class Meta + { + public int status { get; set; } + public string msg { get; set; } + } + + public class Next + { + public string href { get; set; } + public string method { get; set; } + public QueryParams query_params { get; set; } + } + + public class Note + { + public string type { get; set; } + public int timestamp { get; set; } + public string blog_name { get; set; } + public string blog_uuid { get; set; } + public string blog_url { get; set; } + public bool followed { get; set; } + public string avatar_shape { get; set; } + public AvatarUrl avatar_url { get; set; } + public string post_id { get; set; } + public string reblog_parent_blog_name { get; set; } + } + + public class QueryParams + { + public string mode { get; set; } + public string id { get; set; } + public string before_timestamp { get; set; } + } + + public class Response + { + public List notes { get; set; } + public int total_notes { get; set; } + public bool is_subscribed { get; set; } + public bool can_subscribe { get; set; } + public bool can_hide_or_delete_notes { get; set; } + public bool conversational_notifications_enabled { get; set; } + public Links _links { get; set; } + } + + public class Root + { + public Meta meta { get; set; } + public Response response { get; set; } + + public string statusCode { get; set; } + } +} diff --git a/URLNotesGrabberCORE/TL.db b/URLNotesGrabberCORE/TL.db new file mode 100644 index 0000000000000000000000000000000000000000..809638a86a6a7bdffa4dc160cb19353b69db0f54 GIT binary patch literal 28672 zcmeI%&u-H&9KdmB>sneV>uwPTBrD>uN@1enj6}DEQBkUq1&JO|6|^uawac~y3Ah#R zdkS8MN8mj;V*j9j0@@xRd?VF%e@-0#K28piPhJitMXKLrSF@zhkJOHG9Q8zNrG9)< zN;S-HaojhJ+TxF?owfE2O*6*c=S#Q!Q`OyT)jn^3-yXFO-0SVV8!jM>00IagfB*sr zAbd4k^Ltwj&-0uwcZ=lhG(FDpBA23m5QV)s)NyZr5NcmyzTRy${rzcnHcV!z z-;Bf8abP00vw!F}`@=Xq3L_hOmgK{%Nb{qlc%NRSr^`-#7(VU28pQgcIqf*=zvx9H z{X87)uEq@(19n=SPQC6u_8c`iKTSX8ms9iUI8HtknfxEitj1lb8uPPFN#*MUNkwii zMXj8Q#7ru#nP!w_#VVi0{B6(QIoX)Ka&@+3Z^Q9qmYx*J>|&W@AmL9J#%OKKQnsZ# zFEb9-1~Y9tLQuYK$2BOm+fXFGaaXFXH;$Ohe=o0{d@jXC*+gQ0O#aQVH1TD(%w*%X znaHT)u2h@mg}P&3D%Tb}w%|Yj0R#|0009ILKmY**5I_KdstUAhVtoHsb$Q7P0R#|0 z009ILKmY**5I_I{5n%tH2LS;D5I_I{1Q0*~0R#|00DW8EH6}v@5I_I{1Q0*~ z0R#|0009Kp|K}M%009ILKmY**5I_I{1Q0-=`U33#SAUJk5CH@bKmY**5I_I{1Q0*~ FfnP)p!WsYo literal 0 HcmV?d00001 diff --git a/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj b/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj new file mode 100644 index 0000000..346bc35 --- /dev/null +++ b/URLNotesGrabberCORE/URLNotesGrabberCORE.csproj @@ -0,0 +1,34 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + + + + + + + Always + + + PreserveNewest + + + + diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json new file mode 100644 index 0000000..fa01b72 --- /dev/null +++ b/URLNotesGrabberCORE/appsettings.json @@ -0,0 +1,10 @@ +{ + "appSettings": { + "PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\", + "PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt", + "PathOutputJSON": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotesJSON.txt", + "PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt", + "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt", + "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db" + } +} \ No newline at end of file