jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved 429 retries

Browse Source
This commit is contained in:
jim
2024-11-01 08:01:15 -05:00
parent 225934d3b9
commit 5cddf2a427
4 changed files with 156 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+82 -7
View File
@@ -116,7 +116,7 @@ namespace URLNotesGrabberCORE
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
{
try { AddPost(rootBlogName, postID, DBPath); } catch { }
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, DBPath); } catch { }
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -164,7 +164,7 @@ namespace URLNotesGrabberCORE
" Posts ";
if (withoutNotesOnly)
sql += " WHERE HasNotesGathered = 0";
sql += " WHERE HasNotesGathered = 0 and NotFound = 0";
sql += " GROUP BY postID ORDER BY reblogURL, PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -250,7 +250,7 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' ORDER BY BlogName";
sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName";
else
sql = "SELECT * FROM Blogs ORDER BY BlogName";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -308,6 +308,56 @@ namespace URLNotesGrabberCORE
}
public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static int UpdateAPICount(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -382,11 +432,11 @@ namespace URLNotesGrabberCORE
{
myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
if (myDeserializedClass.response.total_notes != 0)
{
DataAccess.AddBlog(blog);
//if (myDeserializedClass.response.total_notes != 0)
//{
//DataAccess.AddBlog(blog);
//DataAccess.AddPost(blog, ID);
}
//}
}
catch (Exception ex)
{
@@ -396,6 +446,31 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription);
myDeserializedClass.statusCode = response.StatusCode.ToString();
bool checkReset = false;
if (myDeserializedClass.statusCode != "NotFound")
{
foreach (var header in response.Headers)
{
Console.WriteLine("{0} - {1}", header.Name, header.Value);
if (checkReset && header.Name.Contains("Reset"))
{
if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString()))
myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString());
}
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
checkReset = true;
else
checkReset = false;
//if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = "))
//{
// string temp = header.ToString().Substring(59);
// int indexOf = temp.IndexOf(',');
// myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf));
//}
}
}
}
}
URLNotesGrabberCORE/Program.cs
+71 -56
View File
@@ -22,61 +22,52 @@ namespace URLNotesGrabberCORE
{
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
}
else if (args[0] == "-n") //Manual test
{
/*var response = */
//var url = await CreateProductAsync(product);
//var t = await GrabNotes();
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
foreach (var note in response.response.notes)
else
switch(args[0])
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
case "-n": //Manual test
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
while (response.response._links != null)
{
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
break;
}
while (response.response._links != null)
{
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
case "-p": //write post's blogs to file
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
break;
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
//var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList();
//notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList();
case "-b": //write blogs to file
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
break;
//using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
//{
// foreach (var note in notes)
// {
// using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
// }
//}
case "-pn": //collect notes from all posts
CollectNotes(settings.GetValue<string>("PathOutput"));
break;
}
else if (args[0] == "-p") //write post's blogs to file
{
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
}
else if (args[0] == "-b") //write blogs to file
{
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
}
else if (args[0] == "-pn") //collect notes from all posts
{
CollectNotes(settings.GetValue<string>("PathOutput"));
}
else if (args[0] == "-br") //collect notes from all posts
{
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
case "-br": //collect notes from all posts
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
break;
}
@@ -118,6 +109,7 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(blog);
sw.WriteLine(blog + ".tumblr.com");
DataAccess.UpdatePostBlogOutput(blog);
}
}
}
@@ -144,7 +136,7 @@ namespace URLNotesGrabberCORE
return false;
}
static async Task GrabNotes(Tuple<string, long> post)
static async Task<string> GrabNotes(Tuple<string, long> post)
{
try
{
@@ -155,9 +147,20 @@ namespace URLNotesGrabberCORE
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
if (response.statusCode == "NotFound")
return;
{
Thread.Sleep(1000);
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
return response.statusCode;
}
if (response.statusCode == "TooManyRequests")
return;
{
for( int s = 0; s <= response.retryInSeconds; s++)
{
Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s);
Thread.Sleep(1000);
}
return response.statusCode;
}
if (response.response != null)
{
@@ -171,12 +174,14 @@ namespace URLNotesGrabberCORE
}
}
if (response.response == null)
throw new Exception("Response is null");
return "FAILURE";
while (response.response != null && response.response._links != null)
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
return "NULL NOTES";
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
{
@@ -186,8 +191,13 @@ namespace URLNotesGrabberCORE
break;
}
}
return "Success";
}
catch(Exception ex) { Console.WriteLine(ex.ToString()); }
return "UNKNOWN";
}
static async void CollectNotes(string outPath)
@@ -208,17 +218,22 @@ namespace URLNotesGrabberCORE
{
foreach (var post in posts)
{
string status;
using RateLimitLease lease = limiter.AttemptAcquire(1);
if (lease.IsAcquired)
{
await GrabNotes(post);
status = await GrabNotes(post);
}
else
{
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return;
}
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
if(status == "Success")
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
else
Console.WriteLine("GrabNotes Result: " + status);
}
}
}
URLNotesGrabberCORE/Properties/launchSettings.json
+1 -1
View File
@@ -2,7 +2,7 @@
"profiles": {
"URLNotesGrabberCORE": {
"commandName": "Project",
"commandLineArgs": "-br"
"commandLineArgs": "-pn"
}
}
}
URLNotesGrabberCORE/ResponseNotes.cs
+2
View File
@@ -77,5 +77,7 @@ namespace URLNotesGrabberCORE
public Response response { get; set; }
public string statusCode { get; set; }
public int retryInSeconds { get; set; }
}
}