diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 64fbfe7..3cde8e0 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -226,7 +226,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)"; + string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@rootBlogName", rootBlogName); @@ -245,8 +245,6 @@ namespace URLNotesGrabberCORE Console.WriteLine(ex.Message); Console.WriteLine("^^^^^ - SHORTCUT"); } - else - return UpdateNote(rootBlogName, noteBlogName, postID, timestamp, type, DBPath); } finally { @@ -298,7 +296,7 @@ namespace URLNotesGrabberCORE Console.WriteLine(withoutNotesOnly); Console.WriteLine(sql); Console.Write(">"); //Console.ReadKey(); - Thread.Sleep(1000); + // Thread.Sleep(1000); using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -790,7 +788,7 @@ namespace URLNotesGrabberCORE if (!string.IsNullOrEmpty(timestamp)) { URL += "&before_timestamp=" + timestamp; - await Task.Delay(1000); + await Task.Delay(100); } var client = new RestClient(URL); var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, @@ -815,6 +813,55 @@ namespace URLNotesGrabberCORE { myDeserializedClass = deserializedResult; myDeserializedClass.rawJson = myJsonResponse; + + // If the response JSON indicates a 429 (Too Many Requests) via meta.status or message, + // treat it like a rate-limited response and attempt to read Retry headers. + bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429; + bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0; + + if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))) + { + // populate retryInSeconds from headers if possible + if (response?.Headers != null) + { + bool checkResetLocal = false; + foreach (var header in response.Headers) + { + string? headerName = header?.Name; + string? headerValue = header?.Value?.ToString(); + if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue)) + continue; + + if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int retrySecs)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs); + else if (DateTimeOffset.TryParse(headerValue, out var dto)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds)); + } + + if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch)) + { + var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs); + } + + if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0") + checkResetLocal = true; + + if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0) + { + if (int.TryParse(headerValue, out int resetValue)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue); + else if (long.TryParse(headerValue, out long epochVal)) + myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds())); + } + } + } + + // surface rate-limit status back to caller + myDeserializedClass.statusCode = "TooManyRequests"; + } } } catch (Exception ex) @@ -824,31 +871,89 @@ namespace URLNotesGrabberCORE if (!response.IsSuccessful) { - Console.WriteLine($"{response.StatusCode}\t{response.StatusDescription}"); - myDeserializedClass.statusCode = response.StatusCode.ToString(); + // Response.StatusCode may be null with some RestSharp responses. + // Guard it and still attempt to extract retry time from headers (Retry-After, Remaining/Reset, X-RateLimit-Reset) + string? statusStr = null; + try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; } + Console.WriteLine($"{statusStr}\t{response?.StatusDescription}"); + // Only set the status if we actually have one; otherwise leave existing value alone (may be null) + if (!string.IsNullOrEmpty(statusStr)) + myDeserializedClass.statusCode = statusStr; + bool checkReset = false; - if (myDeserializedClass.statusCode != "NotFound" && response.Headers != null) + if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null) { + bool foundRateLimitHeader = false; foreach (var header in response.Headers) { if (header.Name != null && header.Value != null) { Console.WriteLine($"{header.Name} - {header.Value}"); - if (checkReset && header.Name.Contains("Reset")) + // Common header patterns used by APIs to indicate retry times: + // - Retry-After: either seconds or HTTP date + // - X-RateLimit-Reset: often seconds since epoch + // - Reset (after Remaining==0): seconds + var headerValue = header.Value?.ToString(); + if (!string.IsNullOrEmpty(headerValue)) { - var headerValue = header.Value.ToString(); - if (headerValue != null && int.TryParse(headerValue, out int resetValue)) + // 1) Retry-After header (seconds or date) + if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase)) { - if (myDeserializedClass.retryInSeconds < resetValue) - myDeserializedClass.retryInSeconds = resetValue; + if (int.TryParse(headerValue, out int retrySecs)) + { + if (myDeserializedClass.retryInSeconds < retrySecs) + myDeserializedClass.retryInSeconds = retrySecs; + } + else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto)) + { + var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + } + + // 2) Common 'Reset' header: numeric seconds or epoch seconds + if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase)) + { + if (int.TryParse(headerValue, out int resetValue)) + { + if (myDeserializedClass.retryInSeconds < resetValue) + myDeserializedClass.retryInSeconds = resetValue; + } + else if (long.TryParse(headerValue, out long epochVal)) + { + // treat as epoch seconds -> compute secs until that epoch + var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + } + + // 3) X-RateLimit-Reset header: often epoch seconds + if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0) + { + if (long.TryParse(headerValue, out long epoch)) + { + var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + if (myDeserializedClass.retryInSeconds < secs) + myDeserializedClass.retryInSeconds = secs; + } + foundRateLimitHeader = true; } } - if (header.Name.Contains("Remaining") && header.Value.ToString() == "0") + if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0") checkReset = true; else checkReset = false; } + + // If we detected rate-limit related headers (Retry-After / X-RateLimit-Reset / Remaining/Reset) + // or the HTTP status indicates 429, surface it. + if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)) + { + myDeserializedClass.statusCode = "TooManyRequests"; + } } } } diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 44d084c..ae80862 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -416,7 +416,8 @@ namespace URLNotesGrabberCORE return "Success"; } - catch(Exception ex) { Console.WriteLine(ex.ToString()); } + catch(Exception ex) { + Console.WriteLine(ex.ToString()); } return "UNKNOWN"; @@ -446,7 +447,7 @@ namespace URLNotesGrabberCORE if (lease.IsAcquired) { Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString()); - Thread.Sleep(1000); + // Thread.Sleep(1000); status = await GrabNotes(post); } else