✅ Captures replies and reblogs with comment
✅ Ignores rollup_notes field (as requested)
✅ Maintains rate limiting and error handling
✅ Console output shows both reply and reblog comment

.gitignore

@@ -361,3 +361,4 @@ MigrationBackup/
 
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
+/URLNotesGrabberCORE/TL.db

RERUN.sqbpro

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="5" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="4" value="&lt;&gt;1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
+<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="C:/Users/jim/Nextcloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="1"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000387fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc00000000000007740000011e00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011e00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="27" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="114"/><column index="4" value="300"/><column index="5" value="61"/><column index="6" value="191"/><column index="7" value="148"/><column index="8" value="66"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="71"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="42"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="35"/><column index="18" value="63"/><column index="19" value="84"/><column index="20" value="300"/><column index="21" value="83"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="35"/><column index="25" value="51"/><column index="26" value="223"/><column index="27" value="300"/><column index="28" value="159"/><column index="29" value="159"/></column_widths><filter_values><column index="27" value="&gt;2026-05-04 22:44:31"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
 SET HasNotesGathered = 0
 WHERE (BlogName, PostID) IN (
     SELECT p.BlogName, p.PostID
@@ -29,7 +29,7 @@ blogname in
 'nudenymph',
 'caylachief'
 
-)</sql><sql name="SQL 3*">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
+)</sql><sql name="SQL 3">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
 from Notes
 where --type like 'r%' and 
 DatetimeCrawled &lt;&gt; '2/12/26 12am'

URLNotesGrabberCORE/DataAccess.cs

@@ -858,7 +858,7 @@ namespace URLNotesGrabberCORE
             return posts;
         }
 
-        public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int limit = 1)
+        public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
         {
             DBPath ??= GetDefaultDbPath();
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -868,18 +868,26 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
+                string sql = @"SELECT DISTINCT P.BlogName, P.PostID, MAX(N.timestamp) as LatestTimestamp
-                             MAX(Notes.timestamp) as LatestTimestamp
+                              FROM Posts P
-                             FROM Notes 
+                              INNER JOIN Notes N ON N.PostID = P.PostID AND N.RootBlogName = P.BlogName
-                             WHERE Notes.type = 'reply' 
+                              WHERE P.NotFound = 0
-                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                                AND N.type = 'reply' 
-                             GROUP BY Notes.RootBlogName, Notes.PostID
+                                AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')
-                             ORDER BY LatestTimestamp ASC
+                              GROUP BY P.BlogName, P.PostID
-                             LIMIT @limit";
+                              ORDER BY LatestTimestamp ASC";
+
+                if (limit.HasValue)
+                {
+                    sql += " LIMIT @limit";
+                }
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
-                    command.Parameters.AddWithValue("@limit", limit);
+                    if (limit.HasValue)
+                    {
+                        command.Parameters.AddWithValue("@limit", limit.Value);
+                    }
                     using (SQLiteDataReader reader = command.ExecuteReader())
                     {
                         while (reader.Read())
@@ -2188,9 +2196,15 @@ namespace URLNotesGrabberCORE
             }
         }
 
-        public static async Task<PostsRoot> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
+        public static async Task<Root> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
         {
-            var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}&notes_info=true&before_timestamp={timestamp}";
+            var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=conversation";
+            URL = URL.Replace("[0]", blog).Replace("[1]", postID.ToString());
+            if (timestamp > 0)
+            {
+                URL += $"&before_timestamp={timestamp}";
+                await Task.Delay(100);
+            }
 
             using (var client = BuildClient(key, URL))
             {
@@ -2199,7 +2213,7 @@ namespace URLNotesGrabberCORE
 
                 var myJsonResponse = response.Content ?? string.Empty;
                 Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
-                var myDeserializedClass = new PostsRoot();
+                var myDeserializedClass = new Root();
 
                 // Check if response is successful and contains JSON
                 if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
@@ -2225,17 +2239,138 @@ namespace URLNotesGrabberCORE
 
                 try
                 {
-                    var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
+                    var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
                     if (deserializedResult != null)
                     {
                         myDeserializedClass = deserializedResult;
                         myDeserializedClass.rawJson = myJsonResponse;
+
+                        bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
+                        bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
+
+                        if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
+                        {
+                            if (response?.Headers != null)
+                            {
+                                bool checkResetLocal = false;
+                                foreach (var header in response.Headers)
+                                {
+                                    string? headerName = header?.Name;
+                                    string? headerValue = header?.Value?.ToString();
+                                    if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
+                                        continue;
+
+                                    if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
+                                    {
+                                        if (int.TryParse(headerValue, out int retrySecs))
+                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
+                                        else if (DateTimeOffset.TryParse(headerValue, out var dto))
+                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
+                                    }
+
+                                    if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
+                                    {
+                                        var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
+                                        myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
+                                    }
+
+                                    if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
+                                        checkResetLocal = true;
+
+                                    if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
+                                    {
+                                        if (int.TryParse(headerValue, out int resetValue))
+                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
+                                        else if (long.TryParse(headerValue, out long epochVal))
+                                            myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
+                                    }
+                                }
+                            }
+
+                            myDeserializedClass.statusCode = "TooManyRequests";
+                        }
                     }
                 }
                 catch (Exception ex)
                 {
                     Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
                     Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
+
+                    if (!response.IsSuccessful)
+                    {
+                        string? statusStr = null;
+                        try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
+                        Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
+                        if (!string.IsNullOrEmpty(statusStr))
+                            myDeserializedClass.statusCode = statusStr;
+
+                        bool checkReset = false;
+
+                        if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
+                        {
+                            bool foundRateLimitHeader = false;
+                            foreach (var header in response.Headers)
+                            {
+                                if (header.Name != null && header.Value != null)
+                                {
+                                    Console.WriteLine($"{header.Name}  -   {header.Value}");
+                                    var headerValue = header.Value?.ToString();
+                                    if (!string.IsNullOrEmpty(headerValue))
+                                    {
+                                        if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
+                                        {
+                                            if (int.TryParse(headerValue, out int retrySecs))
+                                            {
+                                                if (myDeserializedClass.retryInSeconds < retrySecs)
+                                                    myDeserializedClass.retryInSeconds = retrySecs;
+                                            }
+                                            else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
+                                            {
+                                                var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
+                                                if (myDeserializedClass.retryInSeconds < secs)
+                                                    myDeserializedClass.retryInSeconds = secs;
+                                            }
+                                        }
+
+                                        if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
+                                        {
+                                            if (int.TryParse(headerValue, out int resetValue))
+                                            {
+                                                if (myDeserializedClass.retryInSeconds < resetValue)
+                                                    myDeserializedClass.retryInSeconds = resetValue;
+                                            }
+                                            else if (long.TryParse(headerValue, out long epochVal))
+                                            {
+                                                var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
+                                                if (myDeserializedClass.retryInSeconds < secs)
+                                                    myDeserializedClass.retryInSeconds = secs;
+                                            }
+                                        }
+
+                                        if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
+                                        {
+                                            if (long.TryParse(headerValue, out long epoch))
+                                            {
+                                                var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
+                                                if (myDeserializedClass.retryInSeconds < secs)
+                                                    myDeserializedClass.retryInSeconds = secs;
+                                            }
+                                            foundRateLimitHeader = true;
+                                        }
+                                    }
+                                    if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
+                                        checkReset = true;
+                                    else
+                                        checkReset = false;
+                                }
+
+                                if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
+                                {
+                                    myDeserializedClass.statusCode = "TooManyRequests";
+                                }
+                            }
+                        }
+                    }
                 }
 
                 return myDeserializedClass;

URLNotesGrabberCORE/Program.cs

@@ -460,45 +460,33 @@ namespace URLNotesGrabberCORE
                 {
                     int retry = postsResponse.retryInSeconds > 0 ? postsResponse.retryInSeconds : 60;
                     ApiKeyPool.MarkRateLimited(key, retry);
-                    Console.WriteLine($"[Reply Text] Rate limited, will retry with next key");
+                    Console.WriteLine($"[Reply Text] Rate limited for {retry}s, will retry with next key");
                     return;
                 }
 
-                if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
+                if (postsResponse?.response == null || postsResponse.response.notes == null || postsResponse.response.notes.Count == 0)
                 {
-                    Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
+                    Console.WriteLine($"[Reply Text] No notes found in response for {blogName}/{postID}");
                     Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
                     Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
                     if (postsResponse?.response != null)
                     {
-                        Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
+                        Console.WriteLine($"[Reply Text] Notes count: {postsResponse.response.notes?.Count ?? 0}");
-                    }
-                    if (!string.IsNullOrEmpty(postsResponse?.rawJson))
-                    {
-                        Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
                     }
                     
-                    // Mark all replies for this post with '?' to indicate API returned no posts
+                    var previousColor = Console.ForegroundColor;
-                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
+                    Console.ForegroundColor = ConsoleColor.Yellow;
+                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no notes in response");
+                    Console.ForegroundColor = previousColor;
                     DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
                     
                     return;
                 }
 
-                // Get the first (and should be only) post
-                var post = postsResponse.response.posts.FirstOrDefault();
-                if (post?.notes == null)
-                {
-                    Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
-                    return;
-                }
-
-                Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");
-
                 // Update each reply with its text
                 int replyCount = 0;
                 int skippedCount = 0;
-                foreach (var note in post.notes)
+                foreach (var note in postsResponse.response.notes)
                 {
                     if (note.type == "reply")
                     {
@@ -511,7 +499,10 @@ namespace URLNotesGrabberCORE
                             string displayText = note.reply_text.Length > 100 
                                 ? note.reply_text.Substring(0, 100) + "..." 
                                 : note.reply_text;
+                            var previousColor = Console.ForegroundColor;
+                            Console.ForegroundColor = ConsoleColor.Green;
                             Console.WriteLine($"  [{note.blog_name}] {displayText}");
+                            Console.ForegroundColor = previousColor;
                         }
                         else
                         {
@@ -519,6 +510,21 @@ namespace URLNotesGrabberCORE
                             Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
                         }
                     }
+                    else if (note.type == "reblog" && !string.IsNullOrEmpty(note.reply_text))
+                    {
+                        // Handle reblogs with comment
+                        DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
+                        replyCount++;
+                        
+                        // Output the reblog comment being stored
+                        string displayText = note.reply_text.Length > 100 
+                            ? note.reply_text.Substring(0, 100) + "..." 
+                            : note.reply_text;
+                        var previousColor = Console.ForegroundColor;
+                        Console.ForegroundColor = ConsoleColor.Cyan;
+                        Console.WriteLine($"  [{note.blog_name}] {displayText} (reblog comment)");
+                        Console.ForegroundColor = previousColor;
+                    }
                 }
 
                 if (replyCount > 0)
@@ -553,63 +559,54 @@ namespace URLNotesGrabberCORE
                 DataAccess.EnsureReplyTextColumnExists();
 
                 Console.WriteLine("Starting collection of missing reply text...");
-                Console.WriteLine("Processing 10 posts at a time.");
+                Console.WriteLine("Processing replies directly from Tumblr API.");
+                Console.WriteLine();
 
-                int batchSize = 1;  // Process 10 posts per batch
                 int totalProcessedCount = 0;
+                int totalReplies = DataAccess.GetRepliesWithFilledText()?.Count ?? 0;
 
-                RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
+                Console.WriteLine($"[Reply Text] Total replies to process: {totalReplies}");
-                {
+                Console.WriteLine();
-                    PermitLimit = 300,
-                    QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
-                    QueueLimit = 1,
-                    Window = TimeSpan.FromMinutes(1),
-                    SegmentsPerWindow = 60,
-                    AutoReplenishment = true
-                });
 
-                while (true && totalProcessedCount < batchSize)
+                while (totalProcessedCount < totalReplies)
                 {
-                    var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
+                    var batch = DataAccess.GetRepliesWithFilledText();
 
-                    if (postsWithFilledReplies.Count == 0)
+                    if (batch is null || batch.Count == 0)
                     {
-                        Console.WriteLine("No more posts with filled reply text found. Collection complete.");
+                        Console.WriteLine("[Reply Text] No more posts with missing reply text found.");
                         break;
                     }
 
-                    Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
+                    foreach (var reply in batch)
-                    int batchProcessedCount = 0;
+                    {
+                        var blogName = (reply as Tuple<string, long, long>).Item1;
+                        var postID = (reply as Tuple<string, long, long>).Item2;
+                        var timestamp = (reply as Tuple<string, long, long>).Item3;
 
-                    foreach (var post in postsWithFilledReplies)
+                        await FetchAndStoreReplyText(blogName, postID, timestamp);
-                    {
-                        using RateLimitLease lease = limiter.AttemptAcquire(1);
-                        if (lease.IsAcquired)
-                        {
-                            Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
-                            await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
-                            batchProcessedCount++;
                         totalProcessedCount++;
 
-                            // Add 1 second delay between attempts
+                        int remaining = totalReplies - totalProcessedCount;
-                            await Task.Delay(1000);
+                        double completionPct = (totalProcessedCount / (double)totalReplies) * 100.0;
-                        }
+
-                        else
+                        var previousColor = Console.ForegroundColor;
+                        Console.ForegroundColor = ConsoleColor.Cyan;
+                        Console.WriteLine($"[{remaining} remaining] [{totalProcessedCount}/{totalReplies}] ({completionPct:F1}%)");
+                        Console.ForegroundColor = previousColor;
+
+                        if (totalProcessedCount % 50 == 0 && totalProcessedCount < totalReplies)
                         {
-                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
+                            await Task.Delay(2000);
-                            Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
+                        }
-                            return;
                     }
                 }
 
-                    Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
+                Console.WriteLine($"[Reply Text] Complete. Total processed: {totalProcessedCount} replies.");
-                }
-
-                Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
             }
             catch (Exception ex)
             {
-                Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
+                Console.WriteLine($"[Reply Text] Error: {ex.Message}");
                 Console.WriteLine(ex.ToString());
             }
         }

URLNotesGrabberCORE/ResponseNotes.cs

@@ -5,12 +5,12 @@ using System.Text;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
 
-
 namespace URLNotesGrabberCORE
 {
     internal class ResponseNotes
     {
     }
+
     // Root myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
     public class AvatarUrl
     {
@@ -42,7 +42,7 @@ namespace URLNotesGrabberCORE
     public class Note
     {
         public string type { get; set; }
-        public int timestamp { get; set; }
+        public long timestamp { get; set; }
         public string blog_name { get; set; }
         public string blog_uuid { get; set; }
         public string blog_url { get; set; }
@@ -51,6 +51,7 @@ namespace URLNotesGrabberCORE
         public AvatarUrl avatar_url { get; set; }
         public string post_id { get; set; }
         public string reblog_parent_blog_name { get; set; }
+        public string reply_text { get; set; }
     }
 
     public class QueryParams

URLNotesGrabberCORE/appsettings.json

@@ -1,4 +1,4 @@
-{
+{
   "appSettings": {
     "PathInput": "u:\\jim\\Documents\\Web Copies\\blogs\\",
     "PathOutput": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotes.txt",
@@ -18,7 +18,8 @@
     "ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG",
     "OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J",
     "OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w",
-    "PoolEnabled": true
+    "PoolEnabled": true,
+    "Color": "Cyan"
   },
   "TumblrApi3": {
     "ConsumerKey": "Jmoh13AS9hKBYsf939uENQsiWUuZJLFT3do4YK9P0bpspVuxkH",
@@ -32,6 +33,7 @@
     "ConsumerSecret": "0my913slXAHgra4mEEYV98stJ3wXGjPTWJeviiZXwJ2AqzYtIF   ",
     "OAuthToken": "JHk08YufSrsetTR1Ekuh10cPOzC6rjqU5WwVu3uV4vvLzp0w8w",
     "OAuthTokenSecret": "1oryr4WMBxuDrt4vL4sJQs5qNagkH3KxwIf61M8SfvklTrhHSb",
-    "PoolEnabled": true
+    "PoolEnabled": true,
+    "Color": "Yellow"
   }
 }

vibe.bat

@@ -0,0 +1 @@
+ollama launch opencode --model qwen-coder-32k