fix: make reply text collection process all remaining posts without limit
This commit is contained in:
@@ -858,7 +858,7 @@ namespace URLNotesGrabberCORE
|
|||||||
return posts;
|
return posts;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int limit = 1)
|
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
|
||||||
{
|
{
|
||||||
DBPath ??= GetDefaultDbPath();
|
DBPath ??= GetDefaultDbPath();
|
||||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
@@ -874,12 +874,19 @@ namespace URLNotesGrabberCORE
|
|||||||
WHERE Notes.type = 'reply'
|
WHERE Notes.type = 'reply'
|
||||||
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
|
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
|
||||||
GROUP BY Notes.RootBlogName, Notes.PostID
|
GROUP BY Notes.RootBlogName, Notes.PostID
|
||||||
ORDER BY LatestTimestamp ASC
|
ORDER BY LatestTimestamp ASC";
|
||||||
LIMIT @limit";
|
|
||||||
|
if (limit.HasValue)
|
||||||
|
{
|
||||||
|
sql += " LIMIT @limit";
|
||||||
|
}
|
||||||
|
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
command.Parameters.AddWithValue("@limit", limit);
|
if (limit.HasValue)
|
||||||
|
{
|
||||||
|
command.Parameters.AddWithValue("@limit", limit.Value);
|
||||||
|
}
|
||||||
using (SQLiteDataReader reader = command.ExecuteReader())
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
||||||
{
|
{
|
||||||
while (reader.Read())
|
while (reader.Read())
|
||||||
|
|||||||
@@ -559,7 +559,6 @@ namespace URLNotesGrabberCORE
|
|||||||
Console.WriteLine("Starting collection of missing reply text...");
|
Console.WriteLine("Starting collection of missing reply text...");
|
||||||
Console.WriteLine("Processing 10 posts at a time.");
|
Console.WriteLine("Processing 10 posts at a time.");
|
||||||
|
|
||||||
int batchSize = 1; // Process 10 posts per batch
|
|
||||||
int totalProcessedCount = 0;
|
int totalProcessedCount = 0;
|
||||||
|
|
||||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||||
@@ -572,9 +571,9 @@ namespace URLNotesGrabberCORE
|
|||||||
AutoReplenishment = true
|
AutoReplenishment = true
|
||||||
});
|
});
|
||||||
|
|
||||||
while (true && totalProcessedCount < batchSize)
|
while (true)
|
||||||
{
|
{
|
||||||
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
|
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText();
|
||||||
|
|
||||||
if (postsWithFilledReplies.Count == 0)
|
if (postsWithFilledReplies.Count == 0)
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user