jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DataAccess: use INSERT OR IGNORE for Notes to handle duplicate entries; improve rate limit handling in API responses

Browse Source
This commit is contained in:
jim
2025-11-25 10:59:00 -06:00
parent 30f127cca4
commit e55d2b0e29
2 changed files with 122 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+119 -14
View File
@@ -226,7 +226,7 @@ namespace URLNotesGrabberCORE
{ {
connection.Open(); connection.Open();
string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)"; string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection)) using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
command.Parameters.AddWithValue("@rootBlogName", rootBlogName); command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
@@ -245,8 +245,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
Console.WriteLine("^^^^^ - SHORTCUT"); Console.WriteLine("^^^^^ - SHORTCUT");
} }
else
return UpdateNote(rootBlogName, noteBlogName, postID, timestamp, type, DBPath);
} }
finally finally
{ {
@@ -298,7 +296,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(withoutNotesOnly); Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql); Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey(); Console.Write(">"); //Console.ReadKey();
Thread.Sleep(1000); // Thread.Sleep(1000);
using (SQLiteCommand command = new SQLiteCommand(sql, connection)) using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
@@ -790,7 +788,7 @@ namespace URLNotesGrabberCORE
if (!string.IsNullOrEmpty(timestamp)) if (!string.IsNullOrEmpty(timestamp))
{ {
URL += "&before_timestamp=" + timestamp; URL += "&before_timestamp=" + timestamp;
await Task.Delay(1000); await Task.Delay(100);
} }
var client = new RestClient(URL); var client = new RestClient(URL);
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
@@ -815,6 +813,55 @@ namespace URLNotesGrabberCORE
{ {
myDeserializedClass = deserializedResult; myDeserializedClass = deserializedResult;
myDeserializedClass.rawJson = myJsonResponse; myDeserializedClass.rawJson = myJsonResponse;
// If the response JSON indicates a 429 (Too Many Requests) via meta.status or message,
// treat it like a rate-limited response and attempt to read Retry headers.
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
{
// populate retryInSeconds from headers if possible
if (response?.Headers != null)
{
bool checkResetLocal = false;
foreach (var header in response.Headers)
{
string? headerName = header?.Name;
string? headerValue = header?.Value?.ToString();
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
continue;
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int retrySecs))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
else if (DateTimeOffset.TryParse(headerValue, out var dto))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
}
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
{
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
}
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
checkResetLocal = true;
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (int.TryParse(headerValue, out int resetValue))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
else if (long.TryParse(headerValue, out long epochVal))
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
}
}
}
// surface rate-limit status back to caller
myDeserializedClass.statusCode = "TooManyRequests";
}
} }
} }
catch (Exception ex) catch (Exception ex)
@@ -824,31 +871,89 @@ namespace URLNotesGrabberCORE
if (!response.IsSuccessful) if (!response.IsSuccessful)
{ {
Console.WriteLine($"{response.StatusCode}\t{response.StatusDescription}"); // Response.StatusCode may be null with some RestSharp responses.
myDeserializedClass.statusCode = response.StatusCode.ToString(); // Guard it and still attempt to extract retry time from headers (Retry-After, Remaining/Reset, X-RateLimit-Reset)
string? statusStr = null;
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
// Only set the status if we actually have one; otherwise leave existing value alone (may be null)
if (!string.IsNullOrEmpty(statusStr))
myDeserializedClass.statusCode = statusStr;
bool checkReset = false; bool checkReset = false;
if (myDeserializedClass.statusCode != "NotFound" && response.Headers != null) if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
{ {
bool foundRateLimitHeader = false;
foreach (var header in response.Headers) foreach (var header in response.Headers)
{ {
if (header.Name != null && header.Value != null) if (header.Name != null && header.Value != null)
{ {
Console.WriteLine($"{header.Name} - {header.Value}"); Console.WriteLine($"{header.Name} - {header.Value}");
if (checkReset && header.Name.Contains("Reset")) // Common header patterns used by APIs to indicate retry times:
// - Retry-After: either seconds or HTTP date
// - X-RateLimit-Reset: often seconds since epoch
// - <something>Reset (after Remaining==0): seconds
var headerValue = header.Value?.ToString();
if (!string.IsNullOrEmpty(headerValue))
{ {
var headerValue = header.Value.ToString(); // 1) Retry-After header (seconds or date)
if (headerValue != null && int.TryParse(headerValue, out int resetValue)) if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
{ {
if (myDeserializedClass.retryInSeconds < resetValue) if (int.TryParse(headerValue, out int retrySecs))
myDeserializedClass.retryInSeconds = resetValue; {
if (myDeserializedClass.retryInSeconds < retrySecs)
myDeserializedClass.retryInSeconds = retrySecs;
}
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
{
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
}
// 2) Common 'Reset' header: numeric seconds or epoch seconds
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
{
if (int.TryParse(headerValue, out int resetValue))
{
if (myDeserializedClass.retryInSeconds < resetValue)
myDeserializedClass.retryInSeconds = resetValue;
}
else if (long.TryParse(headerValue, out long epochVal))
{
// treat as epoch seconds -> compute secs until that epoch
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
}
// 3) X-RateLimit-Reset header: often epoch seconds
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (long.TryParse(headerValue, out long epoch))
{
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
foundRateLimitHeader = true;
} }
} }
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0") if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
checkReset = true; checkReset = true;
else else
checkReset = false; checkReset = false;
} }
// If we detected rate-limit related headers (Retry-After / X-RateLimit-Reset / Remaining/Reset)
// or the HTTP status indicates 429, surface it.
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
{
myDeserializedClass.statusCode = "TooManyRequests";
}
} }
} }
} }
URLNotesGrabberCORE/Program.cs
+3 -2
View File
@@ -416,7 +416,8 @@ namespace URLNotesGrabberCORE
return "Success"; return "Success";
} }
catch(Exception ex) { Console.WriteLine(ex.ToString()); } catch(Exception ex) {
Console.WriteLine(ex.ToString()); }
return "UNKNOWN"; return "UNKNOWN";
@@ -446,7 +447,7 @@ namespace URLNotesGrabberCORE
if (lease.IsAcquired) if (lease.IsAcquired)
{ {
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString()); Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
Thread.Sleep(1000); // Thread.Sleep(1000);
status = await GrabNotes(post); status = await GrabNotes(post);
} }
else else