DataAccess: use INSERT OR IGNORE for Notes to handle duplicate entries; improve rate limit handling in API responses
This commit is contained in:
@@ -226,7 +226,7 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)";
|
||||
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)";
|
||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||
{
|
||||
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
||||
@@ -245,8 +245,6 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine(ex.Message);
|
||||
Console.WriteLine("^^^^^ - SHORTCUT");
|
||||
}
|
||||
else
|
||||
return UpdateNote(rootBlogName, noteBlogName, postID, timestamp, type, DBPath);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -298,7 +296,7 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine(withoutNotesOnly);
|
||||
Console.WriteLine(sql);
|
||||
Console.Write(">"); //Console.ReadKey();
|
||||
Thread.Sleep(1000);
|
||||
// Thread.Sleep(1000);
|
||||
|
||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||
{
|
||||
@@ -790,7 +788,7 @@ namespace URLNotesGrabberCORE
|
||||
if (!string.IsNullOrEmpty(timestamp))
|
||||
{
|
||||
URL += "&before_timestamp=" + timestamp;
|
||||
await Task.Delay(1000);
|
||||
await Task.Delay(100);
|
||||
}
|
||||
var client = new RestClient(URL);
|
||||
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
||||
@@ -815,6 +813,55 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
myDeserializedClass = deserializedResult;
|
||||
myDeserializedClass.rawJson = myJsonResponse;
|
||||
|
||||
// If the response JSON indicates a 429 (Too Many Requests) via meta.status or message,
|
||||
// treat it like a rate-limited response and attempt to read Retry headers.
|
||||
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
||||
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
||||
{
|
||||
// populate retryInSeconds from headers if possible
|
||||
if (response?.Headers != null)
|
||||
{
|
||||
bool checkResetLocal = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
string? headerName = header?.Name;
|
||||
string? headerValue = header?.Value?.ToString();
|
||||
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
||||
continue;
|
||||
|
||||
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
||||
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
||||
}
|
||||
|
||||
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
||||
}
|
||||
|
||||
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
||||
checkResetLocal = true;
|
||||
|
||||
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// surface rate-limit status back to caller
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -824,31 +871,89 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
if (!response.IsSuccessful)
|
||||
{
|
||||
Console.WriteLine($"{response.StatusCode}\t{response.StatusDescription}");
|
||||
myDeserializedClass.statusCode = response.StatusCode.ToString();
|
||||
// Response.StatusCode may be null with some RestSharp responses.
|
||||
// Guard it and still attempt to extract retry time from headers (Retry-After, Remaining/Reset, X-RateLimit-Reset)
|
||||
string? statusStr = null;
|
||||
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
||||
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
||||
// Only set the status if we actually have one; otherwise leave existing value alone (may be null)
|
||||
if (!string.IsNullOrEmpty(statusStr))
|
||||
myDeserializedClass.statusCode = statusStr;
|
||||
|
||||
bool checkReset = false;
|
||||
|
||||
if (myDeserializedClass.statusCode != "NotFound" && response.Headers != null)
|
||||
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
||||
{
|
||||
bool foundRateLimitHeader = false;
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
if (header.Name != null && header.Value != null)
|
||||
{
|
||||
Console.WriteLine($"{header.Name} - {header.Value}");
|
||||
if (checkReset && header.Name.Contains("Reset"))
|
||||
// Common header patterns used by APIs to indicate retry times:
|
||||
// - Retry-After: either seconds or HTTP date
|
||||
// - X-RateLimit-Reset: often seconds since epoch
|
||||
// - <something>Reset (after Remaining==0): seconds
|
||||
var headerValue = header.Value?.ToString();
|
||||
if (!string.IsNullOrEmpty(headerValue))
|
||||
{
|
||||
var headerValue = header.Value.ToString();
|
||||
if (headerValue != null && int.TryParse(headerValue, out int resetValue))
|
||||
// 1) Retry-After header (seconds or date)
|
||||
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < resetValue)
|
||||
myDeserializedClass.retryInSeconds = resetValue;
|
||||
if (int.TryParse(headerValue, out int retrySecs))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < retrySecs)
|
||||
myDeserializedClass.retryInSeconds = retrySecs;
|
||||
}
|
||||
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
||||
{
|
||||
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Common 'Reset' header: numeric seconds or epoch seconds
|
||||
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (int.TryParse(headerValue, out int resetValue))
|
||||
{
|
||||
if (myDeserializedClass.retryInSeconds < resetValue)
|
||||
myDeserializedClass.retryInSeconds = resetValue;
|
||||
}
|
||||
else if (long.TryParse(headerValue, out long epochVal))
|
||||
{
|
||||
// treat as epoch seconds -> compute secs until that epoch
|
||||
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
}
|
||||
|
||||
// 3) X-RateLimit-Reset header: often epoch seconds
|
||||
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
if (long.TryParse(headerValue, out long epoch))
|
||||
{
|
||||
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
foundRateLimitHeader = true;
|
||||
}
|
||||
}
|
||||
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
|
||||
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
||||
checkReset = true;
|
||||
else
|
||||
checkReset = false;
|
||||
}
|
||||
|
||||
// If we detected rate-limit related headers (Retry-After / X-RateLimit-Reset / Remaining/Reset)
|
||||
// or the HTTP status indicates 429, surface it.
|
||||
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
||||
{
|
||||
myDeserializedClass.statusCode = "TooManyRequests";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -416,7 +416,8 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
return "Success";
|
||||
}
|
||||
catch(Exception ex) { Console.WriteLine(ex.ToString()); }
|
||||
catch(Exception ex) {
|
||||
Console.WriteLine(ex.ToString()); }
|
||||
|
||||
|
||||
return "UNKNOWN";
|
||||
@@ -446,7 +447,7 @@ namespace URLNotesGrabberCORE
|
||||
if (lease.IsAcquired)
|
||||
{
|
||||
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
|
||||
Thread.Sleep(1000);
|
||||
// Thread.Sleep(1000);
|
||||
status = await GrabNotes(post);
|
||||
}
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user