jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved 429 retries

Browse Source
This commit is contained in:
jim
2024-11-01 08:01:15 -05:00
parent 225934d3b9
commit 5cddf2a427
4 changed files with 156 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+82 -7
View File
@@ -116,7 +116,7 @@ namespace URLNotesGrabberCORE
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
{
try { AddPost(rootBlogName, postID, DBPath); } catch { }
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, DBPath); } catch { }
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -164,7 +164,7 @@ namespace URLNotesGrabberCORE
" Posts ";
if (withoutNotesOnly)
sql += " WHERE HasNotesGathered = 0";
sql += " WHERE HasNotesGathered = 0 and NotFound = 0";
sql += " GROUP BY postID ORDER BY reblogURL, PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -250,7 +250,7 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' ORDER BY BlogName";
sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName";
else
sql = "SELECT * FROM Blogs ORDER BY BlogName";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -308,6 +308,56 @@ namespace URLNotesGrabberCORE
}
public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static int UpdateAPICount(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -382,11 +432,11 @@ namespace URLNotesGrabberCORE
{
myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
if (myDeserializedClass.response.total_notes != 0)
{
DataAccess.AddBlog(blog);
//if (myDeserializedClass.response.total_notes != 0)
//{
//DataAccess.AddBlog(blog);
//DataAccess.AddPost(blog, ID);
}
//}
}
catch (Exception ex)
{
@@ -396,6 +446,31 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription);
myDeserializedClass.statusCode = response.StatusCode.ToString();
bool checkReset = false;
if (myDeserializedClass.statusCode != "NotFound")
{
foreach (var header in response.Headers)
{
Console.WriteLine("{0} - {1}", header.Name, header.Value);
if (checkReset && header.Name.Contains("Reset"))
{
if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString()))
myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString());
}
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
checkReset = true;
else
checkReset = false;
//if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = "))
//{
// string temp = header.ToString().Substring(59);
// int indexOf = temp.IndexOf(',');
// myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf));
//}
}
}
}
}