diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index c0c4e67..c57ef16 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -116,7 +116,7 @@ namespace URLNotesGrabberCORE public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db") { - try { AddPost(rootBlogName, postID, DBPath); } catch { } + //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, DBPath); } catch { } SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -164,7 +164,7 @@ namespace URLNotesGrabberCORE " Posts "; if (withoutNotesOnly) - sql += " WHERE HasNotesGathered = 0"; + sql += " WHERE HasNotesGathered = 0 and NotFound = 0"; sql += " GROUP BY postID ORDER BY reblogURL, PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) @@ -250,7 +250,7 @@ namespace URLNotesGrabberCORE connection.Open(); string sql = ""; if (reblogsOnly) - sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' ORDER BY BlogName"; + sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName"; else sql = "SELECT * FROM Blogs ORDER BY BlogName"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) @@ -308,6 +308,56 @@ namespace URLNotesGrabberCORE } + public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + + public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'"; + SQLiteCommand command = new SQLiteCommand(sql, connection); + + command.ExecuteNonQuery(); + } + catch (Exception ex) + { + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + Console.WriteLine(ex.Message); + } + finally + { + connection.Close(); + } + } + + public static int UpdateAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -382,11 +432,11 @@ namespace URLNotesGrabberCORE { myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); - if (myDeserializedClass.response.total_notes != 0) - { - DataAccess.AddBlog(blog); + //if (myDeserializedClass.response.total_notes != 0) + //{ + //DataAccess.AddBlog(blog); //DataAccess.AddPost(blog, ID); - } + //} } catch (Exception ex) { @@ -396,6 +446,31 @@ namespace URLNotesGrabberCORE { Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription); myDeserializedClass.statusCode = response.StatusCode.ToString(); + bool checkReset = false; + + if (myDeserializedClass.statusCode != "NotFound") + { + foreach (var header in response.Headers) + { + Console.WriteLine("{0} - {1}", header.Name, header.Value); + if (checkReset && header.Name.Contains("Reset")) + { + if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString())) + myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString()); + } + if (header.Name.Contains("Remaining") && header.Value.ToString() == "0") + checkReset = true; + else + checkReset = false; + + //if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = ")) + //{ + // string temp = header.ToString().Substring(59); + // int indexOf = temp.IndexOf(','); + // myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf)); + //} + } + } } } diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index dd4daec..20fe77c 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -22,61 +22,52 @@ namespace URLNotesGrabberCORE { TraverseDirectory( settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs")); } - else if (args[0] == "-n") //Manual test - { - /*var response = */ - //var url = await CreateProductAsync(product); - //var t = await GrabNotes(); - var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult(); - List> notes = new List>(); - - foreach (var note in response.response.notes) + else + switch(args[0]) { - note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + case "-n": //Manual test + + var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult(); + List> notes = new List>(); + + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + + while (response.response._links != null) + { + response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); + + foreach (var note in response.response.notes) + { + note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; + //notes.Add(new Tuple(note.blog_url, note.type)); + if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) + break; + } + } + break; - } - while (response.response._links != null) - { - response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); + case "-p": //write post's blogs to file + WritePostBlogsToFile(settings.GetValue("PathOutputPosts")); + break; - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455"; - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } - } - //var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList(); - //notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList(); + case "-b": //write blogs to file + WriteBlogsToFile(settings.GetValue("PathOutputBlogs")); + break; - //using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true)) - //{ - // foreach (var note in notes) - // { - // using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true)) - // } - //} + case "-pn": //collect notes from all posts + CollectNotes(settings.GetValue("PathOutput")); + break; - } - else if (args[0] == "-p") //write post's blogs to file - { - WritePostBlogsToFile(settings.GetValue("PathOutputPosts")); - } - else if (args[0] == "-b") //write blogs to file - { - WriteBlogsToFile(settings.GetValue("PathOutputBlogs")); - } - else if (args[0] == "-pn") //collect notes from all posts - { - CollectNotes(settings.GetValue("PathOutput")); - } - else if (args[0] == "-br") //collect notes from all posts - { - WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), true); + case "-br": //collect notes from all posts + WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), true); + break; } @@ -118,6 +109,7 @@ namespace URLNotesGrabberCORE { Console.WriteLine(blog); sw.WriteLine(blog + ".tumblr.com"); + DataAccess.UpdatePostBlogOutput(blog); } } } @@ -144,7 +136,7 @@ namespace URLNotesGrabberCORE return false; } - static async Task GrabNotes(Tuple post) + static async Task GrabNotes(Tuple post) { try { @@ -155,9 +147,20 @@ namespace URLNotesGrabberCORE List> notes = new List>(); if (response.statusCode == "NotFound") - return; + { + Thread.Sleep(1000); + DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); + return response.statusCode; + } if (response.statusCode == "TooManyRequests") - return; + { + for( int s = 0; s <= response.retryInSeconds; s++) + { + Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s); + Thread.Sleep(1000); + } + return response.statusCode; + } if (response.response != null) { @@ -171,12 +174,14 @@ namespace URLNotesGrabberCORE } } if (response.response == null) - throw new Exception("Response is null"); + return "FAILURE"; while (response.response != null && response.response._links != null) { response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); - + if (response.response.notes is null) + return "NULL NOTES"; + Console.WriteLine("Notes\t" + response.response.notes.Count); foreach (var note in response.response.notes) { @@ -186,8 +191,13 @@ namespace URLNotesGrabberCORE break; } } + + return "Success"; } catch(Exception ex) { Console.WriteLine(ex.ToString()); } + + + return "UNKNOWN"; } static async void CollectNotes(string outPath) @@ -208,17 +218,22 @@ namespace URLNotesGrabberCORE { foreach (var post in posts) { + string status; + using RateLimitLease lease = limiter.AttemptAcquire(1); if (lease.IsAcquired) { - await GrabNotes(post); + status = await GrabNotes(post); } else { Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); return; } - DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); + if(status == "Success") + DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); + else + Console.WriteLine("GrabNotes Result: " + status); } } } diff --git a/URLNotesGrabberCORE/Properties/launchSettings.json b/URLNotesGrabberCORE/Properties/launchSettings.json index 1e999bd..65ee0f2 100644 --- a/URLNotesGrabberCORE/Properties/launchSettings.json +++ b/URLNotesGrabberCORE/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "URLNotesGrabberCORE": { "commandName": "Project", - "commandLineArgs": "-br" + "commandLineArgs": "-pn" } } } \ No newline at end of file diff --git a/URLNotesGrabberCORE/ResponseNotes.cs b/URLNotesGrabberCORE/ResponseNotes.cs index f3315e8..92aca94 100644 --- a/URLNotesGrabberCORE/ResponseNotes.cs +++ b/URLNotesGrabberCORE/ResponseNotes.cs @@ -77,5 +77,7 @@ namespace URLNotesGrabberCORE public Response response { get; set; } public string statusCode { get; set; } + + public int retryInSeconds { get; set; } } }