diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index c57ef16..21eee04 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -8,6 +8,7 @@ using RestSharp.Authenticators.OAuth; using RestSharp.Authenticators; using RestSharp; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; namespace URLNotesGrabberCORE { @@ -34,6 +35,14 @@ namespace URLNotesGrabberCORE internal class DataAccess { + public static DateTime UnixTimeStampToDateTime(double unixTimeStamp) + { + // Unix timestamp is seconds past epoch + DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime(); + return dateTime; + } + #region Adds public static void AddBlog(string blogName, string DBPath = @"TL.db") { @@ -50,6 +59,7 @@ namespace URLNotesGrabberCORE SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); + Console.WriteLine("+ " + blogName); } catch (Exception ex) { @@ -63,9 +73,10 @@ namespace URLNotesGrabberCORE } - public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db") + public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db") { try { AddBlog(blogName, DBPath); } catch { } + try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { } SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -73,7 +84,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')"; + string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); @@ -81,7 +92,9 @@ namespace URLNotesGrabberCORE catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + { Console.WriteLine(ex.Message); + } } finally { @@ -119,6 +132,8 @@ namespace URLNotesGrabberCORE //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, DBPath); } catch { } + Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName); + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -148,7 +163,64 @@ namespace URLNotesGrabberCORE #endregion Adds #region Gets - public static List> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db") + public static List> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + List> posts = new List>(); + + try + { + connection.Open(); + + string sql = "SELECT " + + " Posts.BlogName, " + + " Posts.PostID, " + + " Max(Notes.timestamp) as LatestNoteTimestamp " + + "FROM " + + " Posts INNER JOIN " + + " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + + "WHERE NotFound = 0 "; + + if (withoutNotesOnly) + sql += " and HasNotesGathered = 0 "; + + sql += "GROUP BY " + + " Posts.BlogName, Posts.PostID " + + "ORDER BY " + + " Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID"; + + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + Tuple post = default; + string blog = null; + long id = 0; + long timestamp; + timestamp = reader.GetInt64(2); // Assuming Id is the first column + id = reader.GetInt64(1); // Assuming Id is the first column + blog = reader.GetString(0); // Assuming Title is the second column + + post = new Tuple(blog, id, timestamp.ToString()); + posts.Add(post); + } + } + } + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + return posts; + } + public static List> GetReplies(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); @@ -157,16 +229,8 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "SELECT " + - " MIN(blogName) as blogName, " + - " postID " + - "FROM" + - " Posts "; + string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID"; - if (withoutNotesOnly) - sql += " WHERE HasNotesGathered = 0 and NotFound = 0"; - - sql += " GROUP BY postID ORDER BY reblogURL, PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) @@ -176,8 +240,9 @@ namespace URLNotesGrabberCORE Tuple post = default; string blog = null; long id = 0; - id = reader.GetInt64(1); // Assuming Id is the first column - blog = reader.GetString(0); // Assuming Title is the second column + + id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column + blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column post = new Tuple(blog, id); posts.Add(post); @@ -240,7 +305,7 @@ namespace URLNotesGrabberCORE } - public static List GetBlogs(bool reblogsOnly, string DBPath = @"TL.db") + public static List GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); @@ -250,9 +315,9 @@ namespace URLNotesGrabberCORE connection.Open(); string sql = ""; if (reblogsOnly) - sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName"; + sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top; else - sql = "SELECT * FROM Blogs ORDER BY BlogName"; + sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) @@ -291,7 +356,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID; + string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); @@ -333,7 +398,32 @@ namespace URLNotesGrabberCORE } - public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db") + public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + + public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -407,7 +497,7 @@ namespace URLNotesGrabberCORE if (timestamp != null) { URL += "&before_timestamp=" + timestamp; - Thread.Sleep(500); + Thread.Sleep(1000); } var client = new RestClient(URL); var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY, @@ -425,7 +515,7 @@ namespace URLNotesGrabberCORE var response = client.Execute(request); var myJsonResponse = response.Content; - Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount()); + Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount()); Root myDeserializedClass = new Root(); try diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 20fe77c..6819722 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -18,38 +18,45 @@ namespace URLNotesGrabberCORE .Build(); var settings = config.GetSection("appSettings"); + List contains = settings.GetValue("ContainsList").Split(',').ToList(); + if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB { - TraverseDirectory( settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs")); + TraverseDirectory( settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains); } else - switch(args[0]) + switch (args[0]) { - case "-n": //Manual test + case "-n": + #region Manual test - var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult(); - List> notes = new List>(); + string blogName = args[1]; + long postID = long.Parse(args[2]); - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } + var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult(); + List> notes = new List>(); - while (response.response._links != null) - { - response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } - } + while (response.response._links != null) + { + response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); + + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ; + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + } + #endregion break; @@ -62,23 +69,57 @@ namespace URLNotesGrabberCORE break; case "-pn": //collect notes from all posts - CollectNotes(settings.GetValue("PathOutput")); + bool withoutNotesOnly = true; + + if (args[1] is not null) + { + bool.TryParse(args[1], out withoutNotesOnly); + + Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]); + } + CollectNotes(settings.GetValue("PathOutput"), withoutNotesOnly); break; case "-br": //collect notes from all posts WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), true); break; - } + + case "-bo": //collect notes from all posts + int from = 1, to = 999999, top = 100; + + if (args[1] is not null && args[2] is not null && args[3] is not null) + { + from = int.Parse(args[1]); + to = int.Parse(args[2]); + top = int.Parse(args[3]); + } + else + { + Console.WriteLine("--Expected FROM TO--"); + } + WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), false, from, to, top); + break; + + case "-pr": //colection posts with replies + try { System.IO.File.Delete(settings.GetValue("PathOutputReplies")); } catch { } + WriteRepliesToFile(settings.GetValue("PathOutputBlogs"), false); + + break; + + default: + Console.WriteLine("** Unknown Command ** " + args[0]); + break; + } System.Console.WriteLine(":/"); - System.Console.ReadKey(); + //System.Console.ReadKey(); } static void WritePostBlogsToFile(string outPath) { - List> posts = DataAccess.GetPosts(); + List> posts = DataAccess.GetPosts(); using (StreamWriter sw = new StreamWriter(outPath, true)) { @@ -98,9 +139,9 @@ namespace URLNotesGrabberCORE } } - static void WriteBlogsToFile(string outPath, bool reblogsOnly = false) + static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100) { - List blogs = DataAccess.GetBlogs(reblogsOnly); + List blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top); blogs.Sort(); using (StreamWriter sw = new StreamWriter(outPath, true)) @@ -109,24 +150,31 @@ namespace URLNotesGrabberCORE { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); - DataAccess.UpdatePostBlogOutput(blog); + DataAccess.UpdateBlogOutput(blog); } } } - protected static bool ContainsAny(string input) + static void WriteRepliesToFile(string outPath, bool reblogsOnly = false) + { + List> posts = DataAccess.GetReplies(); + + using (StreamWriter sw = new StreamWriter(outPath, true)) + { + foreach (var post in posts) + { + Console.WriteLine("{0}\t{1}", post.Item1, post.Item2); + sw.WriteLine("{0}\t{1}", post.Item1 + ".tumblr.com", post.Item2); + } + } + } + + protected static bool ContainsAny(string input, List contains) { if (string.IsNullOrEmpty(input)) return false; - var items = new List(); - items.Add("zombaee"); - items.Add("zomb-eh"); - items.Add("ahzombae"); - items.Add("thebugandme"); - items.Add("lovingbabybug"); - items.Add("swarthyvillain"); - foreach (string item in items) + foreach (string item in contains) { if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0) { @@ -136,28 +184,28 @@ namespace URLNotesGrabberCORE return false; } - static async Task GrabNotes(Tuple post) + static async Task GrabNotes(Tuple post) { try { int APICount = DataAccess.GetAPICount(); - Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now); - var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult(); + Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount); + var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3).GetAwaiter().GetResult(); List> notes = new List>(); if (response.statusCode == "NotFound") { - Thread.Sleep(1000); + Thread.Sleep(2000); DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); return response.statusCode; } if (response.statusCode == "TooManyRequests") { - for( int s = 0; s <= response.retryInSeconds; s++) + for( int s = 0; s <= response.retryInSeconds; s+=15) { - Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s); - Thread.Sleep(1000); + Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString()); + Thread.Sleep(15000); } return response.statusCode; } @@ -174,13 +222,21 @@ namespace URLNotesGrabberCORE } } if (response.response == null) + { + Console.WriteLine("response.response == null"); return "FAILURE"; + } - while (response.response != null && response.response._links != null) + while ( response.response != null + && response.response._links != null + && long.Parse(response.response._links.next.query_params.before_timestamp) >= long.Parse(post.Item3)) { response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); if (response.response.notes is null) + { + Console.WriteLine("response.response.notes == null"); return "NULL NOTES"; + } Console.WriteLine("Notes\t" + response.response.notes.Count); foreach (var note in response.response.notes) @@ -200,9 +256,9 @@ namespace URLNotesGrabberCORE return "UNKNOWN"; } - static async void CollectNotes(string outPath) + static async void CollectNotes(string outPath, bool withoutNotesOnly = true) { - List> posts = DataAccess.GetPosts(true); + List> posts = DataAccess.GetPosts(withoutNotesOnly); RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { PermitLimit = 300, @@ -244,20 +300,21 @@ namespace URLNotesGrabberCORE } - static void TraverseDirectory(string path, string outPath) + static void TraverseDirectory(string path, string outPath, List contains) { // Get all directories in the current directory and sort them alphabetically var directories = Directory.GetDirectories(path); Array.Sort(directories, StringComparer.InvariantCulture); - using (StreamWriter sw = new StreamWriter(outPath, true)) - { + //using (StreamWriter sw = new StreamWriter(outPath, true)) + //{ //sw.WriteLine("=====" + path); - } + //} + foreach (var directory in directories) { Console.WriteLine("Directory: " + directory); - TraverseDirectory(directory, outPath); // Recursively traverse subdirectories + TraverseDirectory(directory, outPath, contains); // Recursively traverse subdirectories } try { @@ -267,12 +324,12 @@ namespace URLNotesGrabberCORE { if (file.EndsWith(".txt")) { - Console.WriteLine($"=====File: {file}"); + //Console.WriteLine($"=====File: {file}"); - using (StreamWriter sw = new StreamWriter(outPath, true)) - { + //using (StreamWriter sw = new StreamWriter(outPath, true)) + //{ //sw.WriteLine("---" + file); - } + //} try { @@ -312,35 +369,35 @@ namespace URLNotesGrabberCORE if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") { - if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated")) + if (ContainsAny(reblog.downloadedFiles, contains) && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated")) { DirectoryInfo currentDir = new DirectoryInfo(path); if (!headerWasWritten) { headerWasWritten = true; } - Console.WriteLine(reblog.postID); - Console.WriteLine(reblog.date); - Console.WriteLine(reblog.reblogKey); - Console.WriteLine(reblog.reblogURL); - Console.WriteLine(reblog.reblogName); - Console.WriteLine(reblog.downloadedFiles); + //Console.WriteLine(reblog.postID); + //Console.WriteLine(reblog.date); + //Console.WriteLine(reblog.reblogKey); + //Console.WriteLine(reblog.reblogURL); + //Console.WriteLine(reblog.reblogName); + //Console.WriteLine(reblog.downloadedFiles); string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); - DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL ); + DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date ); } } } urls.Sort(); - using (StreamWriter sw = new StreamWriter(outPath, true)) - { - foreach (string line in urls.Distinct()) - { - sw.WriteLine(line); - } - } + //using (StreamWriter sw = new StreamWriter(outPath, true)) + //{ + // foreach (string line in urls.Distinct()) + // { + // sw.WriteLine(line); + // } + //} } catch (Exception e) { diff --git a/URLNotesGrabberCORE/Properties/launchSettings.json b/URLNotesGrabberCORE/Properties/launchSettings.json index 65ee0f2..ca7d423 100644 --- a/URLNotesGrabberCORE/Properties/launchSettings.json +++ b/URLNotesGrabberCORE/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "URLNotesGrabberCORE": { "commandName": "Project", - "commandLineArgs": "-pn" + "commandLineArgs": "-pn true 31" } } } \ No newline at end of file diff --git a/URLNotesGrabberCORE/TL.db b/URLNotesGrabberCORE/TL.db index 809638a..96829e7 100644 Binary files a/URLNotesGrabberCORE/TL.db and b/URLNotesGrabberCORE/TL.db differ diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index fa01b72..512eda3 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -1,10 +1,12 @@ { "appSettings": { - "PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\", - "PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt", - "PathOutputJSON": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotesJSON.txt", + "PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\", + "PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt", + "PathOutputJSON": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotesJSON.txt", "PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt", "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt", - "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db" + "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt", + "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db", + "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot" } } \ No newline at end of file