jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Correct CollectMissingReplyText to use API key rate limiting

Browse Source
This commit is contained in:
jim
2026-05-06 09:16:36 -05:00
parent 9c77ecce4b
commit 1507086269
3 changed files with 30 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RERUN.sqbpro
+2 -2
View File
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="5" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="4" value="&lt;&gt;1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts <?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="C:/Users/jim/Nextcloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="1"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000387fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc00000000000007740000011e00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011e00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="27" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="114"/><column index="4" value="300"/><column index="5" value="61"/><column index="6" value="191"/><column index="7" value="148"/><column index="8" value="66"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="71"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="42"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="35"/><column index="18" value="63"/><column index="19" value="84"/><column index="20" value="300"/><column index="21" value="83"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="35"/><column index="25" value="51"/><column index="26" value="223"/><column index="27" value="300"/><column index="28" value="159"/><column index="29" value="159"/></column_widths><filter_values><column index="27" value="&gt;2026-05-04 22:44:31"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
SET HasNotesGathered = 0 SET HasNotesGathered = 0
WHERE (BlogName, PostID) IN ( WHERE (BlogName, PostID) IN (
SELECT p.BlogName, p.PostID SELECT p.BlogName, p.PostID
@@ -29,7 +29,7 @@ blogname in
'nudenymph', 'nudenymph',
'caylachief' 'caylachief'
)</sql><sql name="SQL 3*">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch') )</sql><sql name="SQL 3">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
from Notes from Notes
where --type like 'r%' and where --type like 'r%' and
DatetimeCrawled &lt;&gt; '2/12/26 12am' DatetimeCrawled &lt;&gt; '2/12/26 12am'
URLNotesGrabberCORE/Program.cs
+22 -36
View File
@@ -460,7 +460,7 @@ namespace URLNotesGrabberCORE
{ {
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60; int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
ApiKeyPool.MarkRateLimited(key, retry); ApiKeyPool.MarkRateLimited(key, retry);
Console.WriteLine($"[Reply Text] Rate limited, will retry with next key"); Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
return; return;
} }
@@ -559,62 +559,48 @@ namespace URLNotesGrabberCORE
DataAccess.EnsureReplyTextColumnExists(); DataAccess.EnsureReplyTextColumnExists();
Console.WriteLine("Starting collection of missing reply text..."); Console.WriteLine("Starting collection of missing reply text...");
Console.WriteLine("Processing 10 posts at a time."); Console.WriteLine("Processing replies directly from Tumblr API.");
Console.WriteLine();
int totalProcessedCount = 0; int totalProcessedCount = 0;
int successCount = 0;
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions // Get posts with missing reply text
{
PermitLimit = 300,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
QueueLimit = 1,
Window = TimeSpan.FromMinutes(1),
SegmentsPerWindow = 60,
AutoReplenishment = true
});
while (true) while (true)
{ {
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(); var batch = DataAccess.GetRepliesWithFilledText();
if (postsWithFilledReplies.Count == 0) if (batch is null || batch.Count == 0)
{ {
Console.WriteLine("No more posts with filled reply text found. Collection complete."); Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
break; break;
} }
Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch..."); Console.WriteLine($"[Reply Text] Found {batch.Count} replies to process");
int batchProcessedCount = 0;
foreach (var post in postsWithFilledReplies) foreach (var reply in batch)
{ {
using RateLimitLease lease = limiter.AttemptAcquire(1); var blogName = (reply as Tuple<string, long, long>).Item1;
if (lease.IsAcquired) var postID = (reply as Tuple<string, long, long>).Item2;
{ var timestamp = (reply as Tuple<string, long, long>).Item3;
Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
batchProcessedCount++;
totalProcessedCount++;
// Add 1 second delay between attempts Console.WriteLine($"[Reply Text] Processing {blogName}/{postID}/{timestamp}");
await Task.Delay(1000);
} await FetchAndStoreReplyText(blogName, postID, timestamp);
else totalProcessedCount++;
if (totalProcessedCount % 10 == 0)
{ {
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); await Task.Delay(2000);
Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
return;
} }
} }
Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
} }
Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts."); Console.WriteLine($"[Reply Text] Completed. Total processed: {totalProcessedCount} posts.");
} }
catch (Exception ex) catch (Exception ex)
{ {
Console.WriteLine($"Error collecting missing reply text: {ex.Message}"); Console.WriteLine($"[Reply Text] Error: {ex.Message}");
Console.WriteLine(ex.ToString()); Console.WriteLine(ex.ToString());
} }
} }
URLNotesGrabberCORE/appsettings.json
+5 -3
View File
@@ -1,4 +1,4 @@
{ {
"appSettings": { "appSettings": {
"PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\", "PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\",
"PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt", "PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt",
@@ -18,7 +18,8 @@
"ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG", "ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG",
"OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J", "OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J",
"OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w", "OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w",
"PoolEnabled": true "PoolEnabled": true,
"Color": "Cyan"
}, },
"TumblrApi3": { "TumblrApi3": {
"ConsumerKey": "Jmoh13AS9hKBYsf939uENQsiWUuZJLFT3do4YK9P0bpspVuxkH", "ConsumerKey": "Jmoh13AS9hKBYsf939uENQsiWUuZJLFT3do4YK9P0bpspVuxkH",
@@ -32,6 +33,7 @@
"ConsumerSecret": "0my913slXAHgra4mEEYV98stJ3wXGjPTWJeviiZXwJ2AqzYtIF ", "ConsumerSecret": "0my913slXAHgra4mEEYV98stJ3wXGjPTWJeviiZXwJ2AqzYtIF ",
"OAuthToken": "JHk08YufSrsetTR1Ekuh10cPOzC6rjqU5WwVu3uV4vvLzp0w8w", "OAuthToken": "JHk08YufSrsetTR1Ekuh10cPOzC6rjqU5WwVu3uV4vvLzp0w8w",
"OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb", "OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb",
"PoolEnabled": true "PoolEnabled": true,
"Color": "Yellow"
} }
} }