jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved 429 retries

Browse Source
This commit is contained in:
jim
2024-11-01 08:01:15 -05:00
parent 225934d3b9
commit 5cddf2a427
4 changed files with 156 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+71 -56
View File
@@ -22,61 +22,52 @@ namespace URLNotesGrabberCORE
{
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
}
else if (args[0] == "-n") //Manual test
{
/*var response = */
//var url = await CreateProductAsync(product);
//var t = await GrabNotes();
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
foreach (var note in response.response.notes)
else
switch(args[0])
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
case "-n": //Manual test
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
while (response.response._links != null)
{
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
break;
}
while (response.response._links != null)
{
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
case "-p": //write post's blogs to file
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
break;
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
//var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList();
//notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList();
case "-b": //write blogs to file
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
break;
//using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
//{
// foreach (var note in notes)
// {
// using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
// }
//}
case "-pn": //collect notes from all posts
CollectNotes(settings.GetValue<string>("PathOutput"));
break;
}
else if (args[0] == "-p") //write post's blogs to file
{
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
}
else if (args[0] == "-b") //write blogs to file
{
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
}
else if (args[0] == "-pn") //collect notes from all posts
{
CollectNotes(settings.GetValue<string>("PathOutput"));
}
else if (args[0] == "-br") //collect notes from all posts
{
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
case "-br": //collect notes from all posts
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
break;
}
@@ -118,6 +109,7 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(blog);
sw.WriteLine(blog + ".tumblr.com");
DataAccess.UpdatePostBlogOutput(blog);
}
}
}
@@ -144,7 +136,7 @@ namespace URLNotesGrabberCORE
return false;
}
static async Task GrabNotes(Tuple<string, long> post)
static async Task<string> GrabNotes(Tuple<string, long> post)
{
try
{
@@ -155,9 +147,20 @@ namespace URLNotesGrabberCORE
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
if (response.statusCode == "NotFound")
return;
{
Thread.Sleep(1000);
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
return response.statusCode;
}
if (response.statusCode == "TooManyRequests")
return;
{
for( int s = 0; s <= response.retryInSeconds; s++)
{
Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s);
Thread.Sleep(1000);
}
return response.statusCode;
}
if (response.response != null)
{
@@ -171,12 +174,14 @@ namespace URLNotesGrabberCORE
}
}
if (response.response == null)
throw new Exception("Response is null");
return "FAILURE";
while (response.response != null && response.response._links != null)
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
return "NULL NOTES";
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
{
@@ -186,8 +191,13 @@ namespace URLNotesGrabberCORE
break;
}
}
return "Success";
}
catch(Exception ex) { Console.WriteLine(ex.ToString()); }
return "UNKNOWN";
}
static async void CollectNotes(string outPath)
@@ -208,17 +218,22 @@ namespace URLNotesGrabberCORE
{
foreach (var post in posts)
{
string status;
using RateLimitLease lease = limiter.AttemptAcquire(1);
if (lease.IsAcquired)
{
await GrabNotes(post);
status = await GrabNotes(post);
}
else
{
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return;
}
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
if(status == "Success")
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
else
Console.WriteLine("GrabNotes Result: " + status);
}
}
}