fix: make reply text collection process all remaining posts without limit
This commit is contained in:
@@ -858,7 +858,7 @@ namespace URLNotesGrabberCORE
|
||||
return posts;
|
||||
}
|
||||
|
||||
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int limit = 1)
|
||||
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
@@ -874,12 +874,19 @@ namespace URLNotesGrabberCORE
|
||||
WHERE Notes.type = 'reply'
|
||||
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
|
||||
GROUP BY Notes.RootBlogName, Notes.PostID
|
||||
ORDER BY LatestTimestamp ASC
|
||||
LIMIT @limit";
|
||||
ORDER BY LatestTimestamp ASC";
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
sql += " LIMIT @limit";
|
||||
}
|
||||
|
||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||
{
|
||||
command.Parameters.AddWithValue("@limit", limit);
|
||||
if (limit.HasValue)
|
||||
{
|
||||
command.Parameters.AddWithValue("@limit", limit.Value);
|
||||
}
|
||||
using (SQLiteDataReader reader = command.ExecuteReader())
|
||||
{
|
||||
while (reader.Read())
|
||||
|
||||
@@ -559,7 +559,6 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine("Starting collection of missing reply text...");
|
||||
Console.WriteLine("Processing 10 posts at a time.");
|
||||
|
||||
int batchSize = 1; // Process 10 posts per batch
|
||||
int totalProcessedCount = 0;
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
@@ -572,9 +571,9 @@ namespace URLNotesGrabberCORE
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
while (true && totalProcessedCount < batchSize)
|
||||
while (true)
|
||||
{
|
||||
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
|
||||
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText();
|
||||
|
||||
if (postsWithFilledReplies.Count == 0)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user