diff --git a/RERUN.sqbpro b/RERUN.sqbpro index 08bed12..5c66787 100644 --- a/RERUN.sqbpro +++ b/RERUN.sqbpro @@ -1,4 +1,4 @@ -
UPDATE Posts +
UPDATE Posts SET HasNotesGathered = 0 WHERE (BlogName, PostID) IN ( SELECT p.BlogName, p.PostID @@ -29,7 +29,7 @@ blogname in 'nudenymph', 'caylachief' -)select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch') +)select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch') from Notes where --type like 'r%' and DatetimeCrawled <> '2/12/26 12am' diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index e297a39..b90de42 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -460,7 +460,7 @@ namespace URLNotesGrabberCORE { int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60; ApiKeyPool.MarkRateLimited(key, retry); - Console.WriteLine($"[Reply Text] Rate limited, will retry with next key"); + Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key"); return; } @@ -559,62 +559,48 @@ namespace URLNotesGrabberCORE DataAccess.EnsureReplyTextColumnExists(); Console.WriteLine("Starting collection of missing reply text..."); - Console.WriteLine("Processing 10 posts at a time."); + Console.WriteLine("Processing replies directly from Tumblr API."); + Console.WriteLine(); int totalProcessedCount = 0; + int successCount = 0; - RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions - { - PermitLimit = 300, - QueueProcessingOrder = QueueProcessingOrder.OldestFirst, - QueueLimit = 1, - Window = TimeSpan.FromMinutes(1), - SegmentsPerWindow = 60, - AutoReplenishment = true - }); - + // Get posts with missing reply text while (true) { - var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(); - - if (postsWithFilledReplies.Count == 0) + var batch = DataAccess.GetRepliesWithFilledText(); + + if (batch is null || batch.Count == 0) { - Console.WriteLine("No more posts with filled reply text found. Collection complete."); + Console.WriteLine("[Reply Text] No more posts with missing reply text found."); break; } - Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch..."); - int batchProcessedCount = 0; + Console.WriteLine($"[Reply Text] Found {batch.Count} replies to process"); - foreach (var post in postsWithFilledReplies) + foreach (var reply in batch) { - using RateLimitLease lease = limiter.AttemptAcquire(1); - if (lease.IsAcquired) - { - Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}"); - await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3); - batchProcessedCount++; - totalProcessedCount++; + var blogName = (reply as Tuple).Item1; + var postID = (reply as Tuple).Item2; + var timestamp = (reply as Tuple).Item3; + + Console.WriteLine($"[Reply Text] Processing {blogName}/{postID}/{timestamp}"); - // Add 1 second delay between attempts - await Task.Delay(1000); - } - else + await FetchAndStoreReplyText(blogName, postID, timestamp); + totalProcessedCount++; + + if (totalProcessedCount % 10 == 0) { - Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); - Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total"); - return; + await Task.Delay(2000); } } - - Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch."); } - Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts."); + Console.WriteLine($"[Reply Text] Completed. Total processed: {totalProcessedCount} posts."); } catch (Exception ex) { - Console.WriteLine($"Error collecting missing reply text: {ex.Message}"); + Console.WriteLine($"[Reply Text] Error: {ex.Message}"); Console.WriteLine(ex.ToString()); } } diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index 0509c0d..413f809 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -1,4 +1,4 @@ -{ +{ "appSettings": { "PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\", "PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt", @@ -18,7 +18,8 @@ "ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG", "OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J", "OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w", - "PoolEnabled": true + "PoolEnabled": true, + "Color": "Cyan" }, "TumblrApi3": { "ConsumerKey": "Jmoh13AS9hKBYsf939uENQsiWUuZJLFT3do4YK9P0bpspVuxkH", @@ -32,6 +33,7 @@ "ConsumerSecret": "0my913slXAHgra4mEEYV98stJ3wXGjPTWJeviiZXwJ2AqzYtIF ", "OAuthToken": "JHk08YufSrsetTR1Ekuh10cPOzC6rjqU5WwVu3uV4vvLzp0w8w", "OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb", - "PoolEnabled": true + "PoolEnabled": true, + "Color": "Yellow" } } \ No newline at end of file