Improved 429 retries

URLNotesGrabberCORE/DataAccess.cs

@@ -116,7 +116,7 @@ namespace URLNotesGrabberCORE
 
         public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
         {
-            try { AddPost(rootBlogName, postID, DBPath); } catch { }
+            //try { AddPost(rootBlogName, postID, DBPath); } catch { }
             try { AddBlog(noteBlogName, DBPath); } catch { }
 
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -164,7 +164,7 @@ namespace URLNotesGrabberCORE
                              " Posts ";
 
                 if (withoutNotesOnly)
-                    sql += " WHERE HasNotesGathered = 0";
+                    sql += " WHERE HasNotesGathered = 0 and NotFound = 0";
 
                 sql += " GROUP BY  postID ORDER BY reblogURL, PostID";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -250,7 +250,7 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog'  ORDER BY BlogName";
+                    sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0  ORDER BY BlogName";
                 else
                     sql = "SELECT * FROM Blogs ORDER BY BlogName";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -308,6 +308,56 @@ namespace URLNotesGrabberCORE
         }
 
 
+        public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
+        public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'";
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
         public static int UpdateAPICount(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -382,11 +432,11 @@ namespace URLNotesGrabberCORE
             {
                 myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
 
-                if (myDeserializedClass.response.total_notes != 0)
+                //if (myDeserializedClass.response.total_notes != 0)
-                {
+                //{
-                    DataAccess.AddBlog(blog);
+                    //DataAccess.AddBlog(blog);
                     //DataAccess.AddPost(blog, ID);
-                }
+                //}
             }
             catch (Exception ex) 
             { 
@@ -396,6 +446,31 @@ namespace URLNotesGrabberCORE
                 {
                     Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription);
                     myDeserializedClass.statusCode = response.StatusCode.ToString();
+                    bool checkReset = false;
+
+                    if (myDeserializedClass.statusCode != "NotFound")
+                    {
+                        foreach (var header in response.Headers)
+                        {
+                            Console.WriteLine("{0}  -   {1}", header.Name, header.Value);
+                            if (checkReset && header.Name.Contains("Reset"))
+                            {
+                                if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString()))
+                                    myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString());
+                            }
+                            if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
+                                checkReset = true;
+                            else
+                                checkReset = false;
+
+                            //if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = "))
+                            //{
+                            //    string temp = header.ToString().Substring(59);
+                            //    int indexOf = temp.IndexOf(',');
+                            //    myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf));
+                            //}
+                        }
+                    }
                 }
             }
 

URLNotesGrabberCORE/Program.cs

@@ -22,61 +22,52 @@ namespace URLNotesGrabberCORE
             {
                 TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
             }
-            else if (args[0] == "-n") //Manual test 
+            else
-            {
+                switch(args[0])
-                /*var response = */
-                //var url = await CreateProductAsync(product);
-                //var t = await GrabNotes();
-                var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
-                List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
-
-                foreach (var note in response.response.notes)
                 {
-                    note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                    case "-n": //Manual test 
-                    //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
+
-                    if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
+                                var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
+                                List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
+
+                                foreach (var note in response.response.notes)
+                                {
+                                    note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                                    //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
+                                    if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
+                                        break;
+                                }
+
+                                while (response.response._links != null)
+                                {
+                                    response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
+
+                                    foreach (var note in response.response.notes)
+                                    {
+                                        note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                                        //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
+                                        if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
+                                            break;
+                                    }
+                                }
+
                         break;
-                }
 
-                while (response.response._links != null)
+                    case "-p": //write post's blogs to file
-                {
+                        WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
-                    response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
+                        break;
 
-                    foreach (var note in response.response.notes)
+                    case "-b": //write blogs to file
-                    {
+                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
-                        note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                        break;
-                        //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                        if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                            break;
-                    }
-                }
-                //var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList();
-                //notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList();
 
-                //using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
+                    case "-pn": //collect notes from all posts
-                //{
+                        CollectNotes(settings.GetValue<string>("PathOutput"));
-                //    foreach (var note in notes)
+                        break;
-                //    {
-                //        using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
-                //    }
-                //}
 
-            }
+                    case "-br": //collect notes from all posts
-            else if (args[0] == "-p") //write post's blogs to file
+                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
-            {
+                        break;
-                WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
-            }
-            else if (args[0] == "-b") //write blogs to file
-            {
-                WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
-            }
-            else if (args[0] == "-pn") //collect notes from all posts
-            {
-                CollectNotes(settings.GetValue<string>("PathOutput"));
-            }
-            else if (args[0] == "-br") //collect notes from all posts
-            {
-                WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
             }
 
 
@@ -118,6 +109,7 @@ namespace URLNotesGrabberCORE
                 {
                     Console.WriteLine(blog);
                     sw.WriteLine(blog + ".tumblr.com");
+                    DataAccess.UpdatePostBlogOutput(blog);
                 }
             }
         }
@@ -144,7 +136,7 @@ namespace URLNotesGrabberCORE
             return false;
         }
 
-        static async Task GrabNotes(Tuple<string, long> post)
+        static async Task<string> GrabNotes(Tuple<string, long> post)
         {
             try
             {
@@ -155,9 +147,20 @@ namespace URLNotesGrabberCORE
                 List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
 
                 if (response.statusCode == "NotFound")
-                    return;
+                {
+                    Thread.Sleep(1000);
+                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                    return response.statusCode;
+                }
                 if (response.statusCode == "TooManyRequests")
-                    return;
+                {
+                    for( int s  = 0; s <= response.retryInSeconds; s++)
+                    {
+                        Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s);
+                        Thread.Sleep(1000);
+                    }
+                    return response.statusCode;
+                }
 
                 if (response.response != null)
                 {
@@ -171,11 +174,13 @@ namespace URLNotesGrabberCORE
                     }
                 }
                 if (response.response == null)
-                    throw new Exception("Response is null");
+                    return "FAILURE";
 
                 while (response.response != null && response.response._links != null)
                 {
                     response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
+                    if (response.response.notes is null)
+                        return "NULL NOTES";
 
                     Console.WriteLine("Notes\t" + response.response.notes.Count);
                     foreach (var note in response.response.notes)
@@ -186,8 +191,13 @@ namespace URLNotesGrabberCORE
                             break;
                     }
                 }
+
+                return "Success";
             }
             catch(Exception ex) { Console.WriteLine(ex.ToString()); }
+
+
+            return "UNKNOWN";
         }
 
         static async void CollectNotes(string outPath)
@@ -208,17 +218,22 @@ namespace URLNotesGrabberCORE
                 {
                     foreach (var post in posts)
                     {
+                        string status;
+
                         using RateLimitLease lease = limiter.AttemptAcquire(1);
                         if (lease.IsAcquired)
                         {
-                            await GrabNotes(post);
+                            status = await GrabNotes(post);
                         }
                         else
                         {
                             Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                             return;
                         }
-                         DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); 
+                        if(status == "Success")
+                            DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
+                        else
+                            Console.WriteLine("GrabNotes Result: " + status);
                     }
                 }
             }

URLNotesGrabberCORE/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "URLNotesGrabberCORE": {
       "commandName": "Project",
-      "commandLineArgs": "-br"
+      "commandLineArgs": "-pn"
     }
   }
 }

URLNotesGrabberCORE/ResponseNotes.cs

@@ -77,5 +77,7 @@ namespace URLNotesGrabberCORE
         public Response response { get; set; }
 
         public string statusCode { get; set; }
+
+        public int retryInSeconds { get; set; }
     }
 }