fix: Correct CollectMissingReplyText to use API key rate limiting
This commit is contained in:
+2
-2
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="5" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="4" value="<>1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="C:/Users/jim/Nextcloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="1"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000387fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc00000000000007740000011e00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011e00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="27" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="114"/><column index="4" value="300"/><column index="5" value="61"/><column index="6" value="191"/><column index="7" value="148"/><column index="8" value="66"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="71"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="42"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="35"/><column index="18" value="63"/><column index="19" value="84"/><column index="20" value="300"/><column index="21" value="83"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="35"/><column index="25" value="51"/><column index="26" value="223"/><column index="27" value="300"/><column index="28" value="159"/><column index="29" value="159"/></column_widths><filter_values><column index="27" value=">2026-05-04 22:44:31"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
|
||||
SET HasNotesGathered = 0
|
||||
WHERE (BlogName, PostID) IN (
|
||||
SELECT p.BlogName, p.PostID
|
||||
@@ -29,7 +29,7 @@ blogname in
|
||||
'nudenymph',
|
||||
'caylachief'
|
||||
|
||||
)</sql><sql name="SQL 3*">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
|
||||
)</sql><sql name="SQL 3">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
|
||||
from Notes
|
||||
where --type like 'r%' and
|
||||
DatetimeCrawled <> '2/12/26 12am'
|
||||
|
||||
@@ -460,7 +460,7 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
|
||||
ApiKeyPool.MarkRateLimited(key, retry);
|
||||
Console.WriteLine($"[Reply Text] Rate limited, will retry with next key");
|
||||
Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -559,62 +559,48 @@ namespace URLNotesGrabberCORE
|
||||
DataAccess.EnsureReplyTextColumnExists();
|
||||
|
||||
Console.WriteLine("Starting collection of missing reply text...");
|
||||
Console.WriteLine("Processing 10 posts at a time.");
|
||||
Console.WriteLine("Processing replies directly from Tumblr API.");
|
||||
Console.WriteLine();
|
||||
|
||||
int totalProcessedCount = 0;
|
||||
int successCount = 0;
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
// Get posts with missing reply text
|
||||
while (true)
|
||||
{
|
||||
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText();
|
||||
var batch = DataAccess.GetRepliesWithFilledText();
|
||||
|
||||
if (postsWithFilledReplies.Count == 0)
|
||||
if (batch is null || batch.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No more posts with filled reply text found. Collection complete.");
|
||||
Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
|
||||
break;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
|
||||
int batchProcessedCount = 0;
|
||||
Console.WriteLine($"[Reply Text] Found {batch.Count} replies to process");
|
||||
|
||||
foreach (var post in postsWithFilledReplies)
|
||||
foreach (var reply in batch)
|
||||
{
|
||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||
if (lease.IsAcquired)
|
||||
{
|
||||
Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
|
||||
await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
|
||||
batchProcessedCount++;
|
||||
var blogName = (reply as Tuple<string, long, long>).Item1;
|
||||
var postID = (reply as Tuple<string, long, long>).Item2;
|
||||
var timestamp = (reply as Tuple<string, long, long>).Item3;
|
||||
|
||||
Console.WriteLine($"[Reply Text] Processing {blogName}/{postID}/{timestamp}");
|
||||
|
||||
await FetchAndStoreReplyText(blogName, postID, timestamp);
|
||||
totalProcessedCount++;
|
||||
|
||||
// Add 1 second delay between attempts
|
||||
await Task.Delay(1000);
|
||||
}
|
||||
else
|
||||
if (totalProcessedCount % 10 == 0)
|
||||
{
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
|
||||
return;
|
||||
await Task.Delay(2000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
|
||||
}
|
||||
|
||||
Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
|
||||
Console.WriteLine($"[Reply Text] Completed. Total processed: {totalProcessedCount} posts.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
|
||||
Console.WriteLine($"[Reply Text] Error: {ex.Message}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"appSettings": {
|
||||
"PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\",
|
||||
"PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt",
|
||||
@@ -18,7 +18,8 @@
|
||||
"ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG",
|
||||
"OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J",
|
||||
"OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w",
|
||||
"PoolEnabled": true
|
||||
"PoolEnabled": true,
|
||||
"Color": "Cyan"
|
||||
},
|
||||
"TumblrApi3": {
|
||||
"ConsumerKey": "Jmoh13AS9hKBYsf939uENQsiWUuZJLFT3do4YK9P0bpspVuxkH",
|
||||
@@ -32,6 +33,7 @@
|
||||
"ConsumerSecret": "0my913slXAHgra4mEEYV98stJ3wXGjPTWJeviiZXwJ2AqzYtIF ",
|
||||
"OAuthToken": "JHk08YufSrsetTR1Ekuh10cPOzC6rjqU5WwVu3uV4vvLzp0w8w",
|
||||
"OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb",
|
||||
"PoolEnabled": true
|
||||
"PoolEnabled": true,
|
||||
"Color": "Yellow"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user