Add support for fetching and storing reply text

- Add database migration and update logic for replyText column in Notes
- Implement batch collection of missing reply text via Tumblr API
- Add new API integration to fetch reply text for replies
- Enhance DataAccess with methods to query/update replyText
- Update CLI: -replies now collects and stores reply text
- Improve logging (archive logs/), error handling, and output
- Remove TL.db from source control

TL.sqbpro

@@ -1,19 +1,24 @@
-<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="1" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Blogs" custom_title="0" dock_id="1" table="4,5:mainBlogs"/><dock_state state="000000ff00000000fd0000000100000002000005f40000031dfc0100000001fb000000160064006f0063006b00420072006f00770073006500310100000000000005f4000000fb00ffffff000002130000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="0" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="48"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1*">SELECT	␍
-	BlogName || '.tumblr.com/post/' || postID,␍
-	␍
-	datetime(NotesGatheredDateTime, 'unixepoch'), *␍
-FROM␍
-	Posts␍
-WHERE␍
-	NotesGatheredDateTime &lt;&gt; 0␍
-ORDER BY␍
-	postdate desc</sql><sql name="SQL 2*">SELECT	␍
-	datetime(TimeStamp, 'unixepoch'),␍
-	RootBlogName || '.tumblr.com/post/' || postid,␍
-	*,␍
-	NoteBlogName || '.tumblr.com'␍
-FROM␍
-	Notes␍
-WHERE	RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin')␍
-ORDER BY␍
-	TimeStamp desc</sql><current_tab id="0"/></tab_sql></sqlb_project>
+<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="" readonly="1" foreign_keys="" case_sensitive_like="" temp_store="" wal_autocheckpoint="" synchronous=""/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/></tab_structure><tab_browse><table title="." custom_title="0" dock_id="4" table="0,0:"/><dock_state state="000000ff00000000fd0000000100000002000005f40000030ffc0100000002fb000000160064006f0063006b00420072006f00770073006500310100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011700ffffff000005f40000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings/></tab_browse><tab_sql><sql name="SQL 1">SELECT	
+	BlogName || '.tumblr.com/post/' || postID,
+	
+	datetime(NotesGatheredDateTime, 'unixepoch'), *
+FROM
+	Posts
+WHERE
+	NotesGatheredDateTime &lt;&gt; 0
+ORDER BY
+	postdate desc</sql><sql name="SQL 2*">SELECT	
+	datetime(TimeStamp, 'unixepoch'),
+	RootBlogName || '.tumblr.com/post/' || N.postid,
+	*,
+	NoteBlogName || '.tumblr.com'
+FROM
+	Notes N␍
+	inner JOIN␍
+	Posts P on P.PostID = N.PostID and P.BlogName = N.RootBlogName
+WHERE	RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')␍
+		and type like 'r%'␍
+		and RootBlogName = 'zomb-eh'␍
+		and P.HasImage = 1
+ORDER BY
+	TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project>

URLNotesGrabberCORE/DataAccess.cs

@@ -81,6 +81,57 @@ namespace URLNotesGrabberCORE
             return dateTime;
         }
 
+        public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                // Check if replyText column exists
+                string checkSql = "PRAGMA table_info(Notes);";
+                using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
+                {
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        bool columnExists = false;
+                        while (reader.Read())
+                        {
+                            string columnName = reader.GetString(1);
+                            if (columnName.Equals("replyText", StringComparison.OrdinalIgnoreCase))
+                            {
+                                columnExists = true;
+                                break;
+                            }
+                        }
+
+                        if (!columnExists)
+                        {
+                            // Add the column if it doesn't exist
+                            connection.Close();
+                            connection.Open();
+
+                            string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';";
+                            using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection))
+                            {
+                                addCommand.ExecuteNonQuery();
+                                Console.WriteLine("[Migration] Added replyText column to Notes table");
+                            }
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
         #region Adds
         public static void AddBlog(string blogName, string DBPath = @"TL.db")
         {
@@ -93,10 +144,14 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "INSERT OR IGNORE INTO Blogs (BlogName) values('" + blogName + "')";
-                SQLiteCommand command = new SQLiteCommand(sql, connection);
-
-                command.ExecuteNonQuery();
+                // Insert BlogName and DateAdded (current UTC datetime)
+                string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded) VALUES (@BlogName, @DateAdded)";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@BlogName", blogName);
+                    command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    command.ExecuteNonQuery();
+                }
                 // Console.WriteLine("+ " + blogName);
             }
             catch (Exception ex)
@@ -145,7 +200,7 @@ namespace URLNotesGrabberCORE
                                 Title,
                                 HasImage
                                 ) VALUES (" +
-                    Q(blogName) + ", " + postID +", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
+                    Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
@@ -156,27 +211,27 @@ namespace URLNotesGrabberCORE
                 {
                     Console.WriteLine(ex.Message);
                     ///
-                        try
-                        {
-                            connection.Open();
+                    try
+                    {
+                        connection.Open();
 
-                            string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
-                            SQLiteCommand command = new SQLiteCommand(sql, connection);
+                        string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
+                        SQLiteCommand command = new SQLiteCommand(sql, connection);
 
-                            command.ExecuteNonQuery();
-                        }
-                        catch (Exception ex2)
+                        command.ExecuteNonQuery();
+                    }
+                    catch (Exception ex2)
+                    {
+                        if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                         {
-                            if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
-                            {
-                                Console.WriteLine(ex2.Message);
-                            }
+                            Console.WriteLine(ex2.Message);
                         }
-                        finally
-                        {
-                            connection.Close();
-                        }
-                        ///
+                    }
+                    finally
+                    {
+                        connection.Close();
+                    }
+                    ///
                 }
             }
             finally
@@ -188,7 +243,7 @@ namespace URLNotesGrabberCORE
 
         public static void AddAPICount(string DBPath = @"TL.db")
         {
-           SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
             try
             {
@@ -226,7 +281,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)";
+                string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
@@ -234,6 +289,7 @@ namespace URLNotesGrabberCORE
                     command.Parameters.AddWithValue("@PostID", postID);
                     command.Parameters.AddWithValue("@TimeStamp", timestamp);
                     command.Parameters.AddWithValue("@Type", type ?? string.Empty);
+                    command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
 
                     command.ExecuteNonQuery();
                 }
@@ -243,7 +299,7 @@ namespace URLNotesGrabberCORE
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
                 {
                     Console.WriteLine(ex.Message);
-                    Console.WriteLine("^^^^^ - SHORTCUT");                    
+                    Console.WriteLine("^^^^^ - SHORTCUT");
                 }
             }
             finally
@@ -262,7 +318,7 @@ namespace URLNotesGrabberCORE
         /// <param name="withoutNotesOnly"></param>
         /// <param name="DBPath"></param>
         /// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
-        public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
+        public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
@@ -271,32 +327,39 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "SELECT " + 
-                             "  Posts.BlogName, " + Environment.NewLine +
+                string sql = "SELECT " +
+                             "  MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
                              "  Posts.PostID, " + Environment.NewLine +
-                             "  Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
+                             " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
                              "  Posts.NotesGatheredDatetime, " + Environment.NewLine +
                              "  CNT.CNT " + Environment.NewLine +
                              "FROM " + Environment.NewLine +
                              "   Posts " + Environment.NewLine +
-                             "      LEFT OUTER JOIN " + Environment.NewLine + 
+                             "      LEFT OUTER JOIN " + Environment.NewLine +
                              "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
                              "      LEFT OUTER JOIN " + Environment.NewLine +
                              "   (      select BlogName, count(PostID) as CNT from Posts group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
-                             "WHERE NotFound = 0 " + Environment.NewLine;
+                             "WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine;
 
                 if (withoutNotesOnly)
-                    sql += " and HasNotesGathered = 0 "  + Environment.NewLine ; 
-                
-                sql +=       "GROUP BY " + Environment.NewLine +
+                    sql += " and HasNotesGathered = 0 " + Environment.NewLine;
+
+                // Filter by NotesGatheredDateTime if beforeDate is provided
+                if (beforeDate.HasValue)
+                {
+                    long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
+                    sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
+                }
+
+                sql += "GROUP BY " + Environment.NewLine +
                              "   Posts.BlogName, Posts.PostID " + Environment.NewLine +
                              "ORDER BY " + Environment.NewLine +
-                             " notesgathereddatetime,  Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
+                             " notesgathereddatetime,  Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine;
 
-                Console.WriteLine(withoutNotesOnly);
-                Console.WriteLine(sql);
-                Console.Write(">"); //Console.ReadKey();
-                // Thread.Sleep(1000);
+                //Console.WriteLine(withoutNotesOnly);
+                //Console.WriteLine(sql);
+                //Console.Write(">"); //Console.ReadKey();
+                //Thread.Sleep(250);
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -332,7 +395,7 @@ namespace URLNotesGrabberCORE
             }
             return posts;
         }
-        
+
         public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -375,6 +438,107 @@ namespace URLNotesGrabberCORE
             return posts;
         }
 
+        public static List<Tuple<string, long>> GetRepliesWithMissingText(string DBPath = @"TL.db", int limit = 50)
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
+
+            try
+            {
+                connection.Open();
+
+                string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
+                             MAX(Notes.timestamp) as LatestTimestamp
+                             FROM Notes 
+                             WHERE Notes.type = 'reply' 
+                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                             GROUP BY Notes.RootBlogName, Notes.PostID
+                             ORDER BY LatestTimestamp ASC
+                             LIMIT @limit";
+
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@limit", limit);
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            Tuple<string, long> post = default;
+                            string blog = null;
+                            long id = 0;
+
+                            blog = reader.GetString(reader.GetOrdinal("blogName"));
+                            id = reader.GetInt64(reader.GetOrdinal("PostID"));
+
+                            post = new Tuple<string, long>(blog, id);
+                            posts.Add(post);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return posts;
+        }
+
+        public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string DBPath = @"TL.db", int limit = 1)
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
+
+            try
+            {
+                connection.Open();
+
+                string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
+                             MAX(Notes.timestamp) as LatestTimestamp
+                             FROM Notes 
+                             WHERE Notes.type = 'reply' 
+                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                             GROUP BY Notes.RootBlogName, Notes.PostID
+                             ORDER BY LatestTimestamp ASC
+                             LIMIT @limit";
+
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@limit", limit);
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            Tuple<string, long, long> post = default;
+                            string blog = null;
+                            long id = 0;
+                            long timestamp = 0;
+
+                            blog = reader.GetString(reader.GetOrdinal("blogName"));
+                            id = reader.GetInt64(reader.GetOrdinal("PostID"));
+                            timestamp = reader.GetInt64(reader.GetOrdinal("LatestTimestamp"));
+                            timestamp++;
+
+                            post = new Tuple<string, long, long>(blog, id, timestamp);
+                            posts.Add(post);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return posts;
+        }
+
         public static int GetAPICount(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -395,7 +559,7 @@ namespace URLNotesGrabberCORE
                     {
                         while (reader.Read())
                         {
-                           count = reader.GetInt32(0); // Assuming Id is the first column
+                            count = reader.GetInt32(0); // Assuming Id is the first column
                         }
                     }
                 }
@@ -510,7 +674,8 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
+                //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
+                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE PostID = @PostID";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
@@ -635,10 +800,10 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql  =  "UPDATE Posts SET ";
+                string sql = "UPDATE Posts SET ";
                 sql += "postDate = @postDate, ";
                 sql += "reblogURL = @reblogURL, ";
-                sql += "postURL = @postURL, "; 
+                sql += "postURL = @postURL, ";
                 sql += "slug = @slug, ";
                 sql += "reblogKey = @reblogKey, ";
                 sql += "reblogName = @reblogName, ";
@@ -752,8 +917,89 @@ namespace URLNotesGrabberCORE
             }
 
             return APICount;
+        }
+
+        public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
+                string sql = "UPDATE Notes SET replyText = @replyText WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@replyText", replyText ?? ".");
+                    command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
+                    command.Parameters.AddWithValue("@PostID", postID);
+                    command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
+                    command.Parameters.AddWithValue("@TimeStamp", timestamp);
+                    int rowsAffected = command.ExecuteNonQuery();
+                    
+                    if (rowsAffected == 0)
+                    {
+                        Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}");
+                        Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
+                        Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
+                    }
+                    else
+                    {
+                        Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}");
+                    }
+                }
             }
-            #endregion Updates
+            catch (Exception ex)
+            {
+                Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
+                Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+        public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@replyText", replyText ?? ".");
+                    command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
+                    command.Parameters.AddWithValue("@PostID", postID);
+                    int rowsAffected = command.ExecuteNonQuery();
+                    
+                    if (rowsAffected == 0)
+                    {
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
+                    }
+                    else
+                    {
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'");
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
+                Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+        #endregion Updates
     }
 
     internal class APIAccess
@@ -772,35 +1018,38 @@ namespace URLNotesGrabberCORE
                 throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json");
         }
 
-        private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ?? 
+        private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
             throw new InvalidOperationException("ConsumerKey is not configured");
-        private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ?? 
+        private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
             throw new InvalidOperationException("ConsumerSecret is not configured");
-        private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ?? 
-            throw new InvalidOperationException("OAuthToken is not configured");
-        private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ?? 
+        private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
+            throw new InvalidOperationException("OAuthToken is not configured"); 
+        private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
             throw new InvalidOperationException("OAuthTokenSecret is not configured");
 
         public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
         {
-            var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]";
+            var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all";
             URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
             if (!string.IsNullOrEmpty(timestamp))
             {
                 URL += "&before_timestamp=" + timestamp;
                 await Task.Delay(100);
             }
-            var client = new RestClient(URL);
-            var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
-                                                            consumerSecret: ConsumerSecret,
-                                                            token: OAuthToken,
-                                                            tokenSecret: OAuthTokenSecret,
-                                                            OAuthSignatureMethod.HmacSha1
-                                                            );
+            
+            // Create a new RestClient for each request to ensure fresh OAuth signatures
+            using (var client = new RestClient(URL))
+            {
+                var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
+                                                                consumerSecret: ConsumerSecret,
+                                                                token: OAuthToken,
+                                                                tokenSecret: OAuthTokenSecret,
+                                                                OAuthSignatureMethod.HmacSha1
+                                                                );
 
-            client.Authenticator = oAuth1;
-            var request = new RestRequest(URL, Method.Get);
-            var response = await client.ExecuteAsync(request);
+                client.Authenticator = oAuth1;
+                var request = new RestRequest(URL, Method.Get);
+                var response = await client.ExecuteAsync(request);
 
             var myJsonResponse = response.Content ?? string.Empty;
             Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
@@ -864,8 +1113,8 @@ namespace URLNotesGrabberCORE
                     }
                 }
             }
-            catch (Exception ex) 
-            { 
+            catch (Exception ex)
+            {
                 Console.WriteLine($"Failed JSON: {myJsonResponse}");
                 Console.WriteLine(ex.ToString());
 
@@ -939,7 +1188,7 @@ namespace URLNotesGrabberCORE
                                             if (myDeserializedClass.retryInSeconds < secs)
                                                 myDeserializedClass.retryInSeconds = secs;
                                         }
-                                    foundRateLimitHeader = true;
+                                        foundRateLimitHeader = true;
                                     }
                                 }
                                 if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
@@ -955,11 +1204,75 @@ namespace URLNotesGrabberCORE
                                 myDeserializedClass.statusCode = "TooManyRequests";
                             }
                         }
-                    }
+                }
                 }
             }
 
             return myDeserializedClass;
+            }
+        }
+
+        public static async Task<PostsRoot> GrabPostWithReplies(string blog, long postID, long timestamp)
+        {
+            var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}&notes_info=true&before_timestamp={timestamp}";
+
+            // Create a new RestClient for each request to ensure fresh OAuth signatures
+            using (var client = new RestClient(URL))
+            {
+                var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
+                                                                consumerSecret: ConsumerSecret,
+                                                                token: OAuthToken,
+                                                                tokenSecret: OAuthTokenSecret,
+                                                                OAuthSignatureMethod.HmacSha1
+                                                                );
+
+                client.Authenticator = oAuth1;
+                var request = new RestRequest(URL, Method.Get);
+                var response = await client.ExecuteAsync(request);
+
+                var myJsonResponse = response.Content ?? string.Empty;
+                Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
+                var myDeserializedClass = new PostsRoot();
+
+                // Check if response is successful and contains JSON
+                if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
+                {
+                    string? statusStr = null;
+                    try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
+                    Console.WriteLine($"[Reply API] {statusStr}\t{response?.StatusDescription}");
+                    if (!string.IsNullOrEmpty(statusStr))
+                        myDeserializedClass.statusCode = statusStr;
+                    return myDeserializedClass;
+                }
+
+                // Check if response is HTML (error page) instead of JSON
+                if (myJsonResponse.TrimStart().StartsWith("<"))
+                {
+                    Console.WriteLine($"[Reply API] Received HTML response (likely error page): {response?.StatusCode}");
+                    string? statusStr = null;
+                    try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
+                    if (!string.IsNullOrEmpty(statusStr))
+                        myDeserializedClass.statusCode = statusStr;
+                    return myDeserializedClass;
+                }
+
+                try
+                {
+                    var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
+                    if (deserializedResult != null)
+                    {
+                        myDeserializedClass = deserializedResult;
+                        myDeserializedClass.rawJson = myJsonResponse;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
+                    Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
+                }
+
+                return myDeserializedClass;
+            }
         }
     }
 }

URLNotesGrabberCORE/Program.cs

@@ -26,9 +26,18 @@ namespace URLNotesGrabberCORE
             if (File.Exists(logPath))
             {
                 string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
-                string newPath = $"console_output_{timestamp}.log";
+                string archiveDirectory = "logs";
+
+                // Ensure the archive directory exists
+                if (!Directory.Exists(archiveDirectory))
+                {
+                    Directory.CreateDirectory(archiveDirectory);
+                }
+
+                string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
                 File.Move(logPath, newPath);
             }
+
             StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
             DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
             Console.SetOut(dualLogger);
@@ -37,7 +46,9 @@ namespace URLNotesGrabberCORE
 
             if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
             {
-                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
+                int postsAdded = 0;
+                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded);
+                Console.WriteLine($"Total posts added: {postsAdded}");
             }
             else
             {
@@ -56,102 +67,25 @@ namespace URLNotesGrabberCORE
 
                         Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
 
-                        Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
+                        Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
 
                         Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
 
                         Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
 
-                        Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
-
-                        break;
-                    case "-parse":
-                        string blogNameToParse = args[1];
-
-                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
+                        Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
 
                         break;
                     
+                    case "-parse":
+                        string blogNameToParse = args[1];
+                        int postsAdded = 0;
+                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse);
+                        Console.WriteLine($"Total posts added: {postsAdded}");
+                        break;
+                    
                     case "-test":
-                        #region Manual test 
-
-                        string blogName = args[1];
-                        long postID = long.Parse(args[2]);
-
-                        var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
-                        List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
-
-                        // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound
-                        if (response?.meta != null && response.meta.status == 404)
-                        {
-                            Console.WriteLine($"ERROR: Post not found - {blogName}/{postID} (meta.status=404)");
-                            DataAccess.UpdatePostMarkNotFound(blogName, postID);
-                            break;
-                        }
-
-                        // Check for API errors
-                        if (response.statusCode == "NotFound")
-                        {
-                            Console.WriteLine($"ERROR: Post not found - {blogName}/{postID}");
-                            break;
-                        }
-                        if (response.statusCode == "TooManyRequests")
-                        {
-                            Console.WriteLine($"ERROR: Rate limited - retry in {response.retryInSeconds} seconds");
-                            break;
-                        }
-                        if (!string.IsNullOrEmpty(response.statusCode))
-                        {
-                            Console.WriteLine($"ERROR: API returned status code: {response.statusCode}");
-                            break;
-                        }
-
-                        // Check for null response
-                        if (response.response == null)
-                        {
-                            Console.WriteLine("ERROR: API response is null");
-                            break;
-                        }
-
-                        // Check for null notes
-                        if (response.response.notes == null)
-                        {
-                            Console.WriteLine("ERROR: Notes collection is null (post may have 0 notes)");
-                            break;
-                        }
-
-                        Console.WriteLine($"Processing {response.response.notes.Count} notes...");
-                        foreach (var note in response.response.notes)
-                        {
-                            note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
-                            //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                            if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                                break;
-                        }
-
-                        while (response.response._links != null)
-                        {
-                            response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
-
-                            if (response.response?.notes == null)
-                            {
-                                Console.WriteLine("ERROR: Notes collection became null during pagination");
-                                break;
-                            }
-
-                            Console.WriteLine($"Processing {response.response.notes.Count} more notes...");
-                            foreach (var note in response.response.notes)
-                            {
-                                note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
-                                //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                                if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                                    break;
-                            }
-                        }
-                        
-                        Console.WriteLine("Test completed successfully");
-                        #endregion
-
+                        Console.WriteLine("Test command not implemented");
                         break;
 
                     case "-post":
@@ -168,13 +102,15 @@ namespace URLNotesGrabberCORE
 
                     case "-collect": //collect notes from all posts
                         bool withoutNotesOnly = true;
+                        DateTime? beforeDate = DateTime.Now;
 
-                        if(args.Length != 2)
+                        if (args.Length < 2)
                         {
-                            Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
+                            Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
                             break;
                         }
 
+                        // Parse withoutNotesOnly flag
                         if (args.Length > 1 && args[1] is not null)
                         {
                             if (args[1] == "1")
@@ -190,7 +126,23 @@ namespace URLNotesGrabberCORE
 
                             Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
                         }
-                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
+
+                        // Parse optional beforeDate parameter
+                        if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
+                        {
+                            if (DateTime.TryParse(args[2], out DateTime parsedDate))
+                            {
+                                beforeDate = parsedDate;
+                                Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
+                            }
+                            else
+                            {
+                                Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
+                                break;
+                            }
+                        }
+
+                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
                         break;
 
                     case "-blogsR": //collect notes from all posts
@@ -229,10 +181,8 @@ namespace URLNotesGrabberCORE
                         WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
                         break;
 
-                    case "-replies": //colection posts with replies
-                        try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
-                        WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
-
+                    case "-replies": //update reply text
+                        CollectMissingReplyText().GetAwaiter().GetResult();
                         break;
 
                     default:
@@ -249,7 +199,7 @@ namespace URLNotesGrabberCORE
 
         static void WritePostBlogsToFile(string outPath)
         {
-            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(); 
+            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
 
             using (StreamWriter sw = new StreamWriter(outPath, true))
             {
@@ -333,16 +283,181 @@ namespace URLNotesGrabberCORE
             return false;
         }
 
+        static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
+        {
+            try
+            {
+                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
+                // Add 2-second delay before API call to avoid server-side rate limiting
+                await Task.Delay(2000);
+                var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp);
+
+                if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
+                {
+                    Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
+                    Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
+                    Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
+                    if (postsResponse?.response != null)
+                    {
+                        Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
+                    }
+                    if (!string.IsNullOrEmpty(postsResponse?.rawJson))
+                    {
+                        Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
+                    }
+                    
+                    // Mark all replies for this post with '?' to indicate API returned no posts
+                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
+                    DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                    
+                    return;
+                }
+
+                // Get the first (and should be only) post
+                var post = postsResponse.response.posts.FirstOrDefault();
+                if (post?.notes == null)
+                {
+                    Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
+                    return;
+                }
+
+                Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");
+
+                // Update each reply with its text
+                int replyCount = 0;
+                int skippedCount = 0;
+                foreach (var note in post.notes)
+                {
+                    if (note.type == "reply")
+                    {
+                        if (!string.IsNullOrEmpty(note.reply_text))
+                        {
+                            DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
+                            replyCount++;
+                            
+                            // Output the reply text being stored
+                            string displayText = note.reply_text.Length > 100 
+                                ? note.reply_text.Substring(0, 100) + "..." 
+                                : note.reply_text;
+                            Console.WriteLine($"  [{note.blog_name}] {displayText}");
+                        }
+                        else
+                        {
+                            skippedCount++;
+                            Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
+                        }
+                    }
+                }
+
+                if (replyCount > 0)
+                {
+                    Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
+                }
+                else
+                {
+                    Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
+                }
+
+                // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
+                Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
+                //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
+                //if (cleanupCount > 0)
+                //{
+                //    Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
+                //}
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}");
+                Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
+            }
+        }
+
+        static async Task CollectMissingReplyText()
+        {
+            try
+            {
+                // Ensure the replyText column exists
+                DataAccess.EnsureReplyTextColumnExists();
+
+                Console.WriteLine("Starting collection of missing reply text...");
+                Console.WriteLine("Processing 10 posts at a time.");
+
+                int batchSize = 1;  // Process 10 posts per batch
+                int totalProcessedCount = 0;
+
+                RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
+                {
+                    PermitLimit = 300,
+                    QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                    QueueLimit = 1,
+                    Window = TimeSpan.FromMinutes(1),
+                    SegmentsPerWindow = 60,
+                    AutoReplenishment = true
+                });
+
+                while (true && totalProcessedCount < batchSize)
+                {
+                    var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
+                    
+                    if (postsWithFilledReplies.Count == 0)
+                    {
+                        Console.WriteLine("No more posts with filled reply text found. Collection complete.");
+                        break;
+                    }
+
+                    Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
+                    int batchProcessedCount = 0;
+
+                    foreach (var post in postsWithFilledReplies)
+                    {
+                        using RateLimitLease lease = limiter.AttemptAcquire(1);
+                        if (lease.IsAcquired)
+                        {
+                            Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
+                            await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
+                            batchProcessedCount++;
+                            totalProcessedCount++;
+                        
+                            // Add 1 second delay between attempts
+                            await Task.Delay(1000);
+                        }
+                        else
+                        {
+                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
+                            Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
+                            return;
+                        }
+                    }
+
+                    Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
+                }
+
+                Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
+                Console.WriteLine(ex.ToString());
+            }
+        }
+
         static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
         {
             try
             {
                 int APICount = DataAccess.GetAPICount();
+                Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
+                Thread.Sleep(3000);
 
-                Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
-                var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
-                
-                // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return
+                var allNotes = new List<dynamic>();
+                int page = 1;
+                string beforeTimestamp = post.Item3.ToString();
+                bool hasReplies = false;
+                const int maxPages = 500;
+                var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
+
+                // Handle 404 and error codes
                 if (response?.meta != null && response.meta.status == 404)
                 {
                     Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)");
@@ -350,8 +465,11 @@ namespace URLNotesGrabberCORE
                     Thread.Sleep(1000);
                     return "NotFound";
                 }
-                List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
-
+                if (response == null)
+                {
+                    Console.WriteLine("##### Response is null - API Failure? ###");
+                    return "FAILURE";
+                }
                 if (response.statusCode == "NotFound")
                 {
                     Thread.Sleep(1000);
@@ -368,86 +486,94 @@ namespace URLNotesGrabberCORE
                     return response.statusCode;
                 }
 
-                if (response.response != null && response.response.notes != null)
+                // Pagination loop
+                while (true)
                 {
-                    Console.WriteLine("Notes\t" + response.response.notes.Count);
-                    foreach (var note in response.response.notes)
-                    {
-                        note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
-                        //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                        if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                            break;
-                    }
-                }
-                else if (response.response == null)
-                {
-                    Console.WriteLine("##### Response Null - API Failure? ###");
-                    Console.WriteLine($"Raw JSON: {response.rawJson}");
-                    return "FAILURE";
-                }
-                else if (response.response.notes == null)
-                {
-                    Console.WriteLine("##### Notes Null - WHY? ###");
-                    return "FAILURE";
-                }
-                else
-                { 
-                   while (response.response != null
-                            && response.response._links != null
-                            && long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
-                    {
-                        response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
-                        if (response.response.notes is null)
-                        {
-                            Console.WriteLine("response.response.notes == null");
-                            return "NULL NOTES";
-                        }
+                    int noteCount = response?.response?.notes?.Count ?? 0;
+                    Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");
 
-                        Console.WriteLine("Notes\t" + response.response.notes.Count);
-                        foreach (var note in response.response.notes)
-                        {
-                            note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
-                            //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                            if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                                break;
-                        }
+                    if (response?.response == null)
+                    {
+                        Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
+                        return "FAILURE";
                     }
+                    if (response.response.notes == null)
+                    {
+                        Console.WriteLine("##### Notes Null - WHY? ###");
+                        return "FAILURE";
+                    }
+
+                    // Accumulate notes
+                    allNotes.AddRange(response.response.notes);
+
+                    // Pagination: check for next
+                    var nextLink = response.response._links?.next;
+                    if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
+                    {
+                        Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
+                        break;
+                    }
+
+                    beforeTimestamp = nextLink.query_params.before_timestamp;
+                    page++;
+                    if (page > maxPages)
+                    {
+                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination.");
+                        break;
+                    }
+
+                    // Fetch next page
+                    response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
+                }
+
+                Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
+
+                // Process all accumulated notes
+                foreach (var note in allNotes)
+                {
+                    note.reblog_parent_blog_name = post.Item1;
+                    note.post_id = post.Item2.ToString();
+                    if (note.type == "reply")
+                        hasReplies = true;
+                    DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
                 }
 
                 return "Success";
             }
-            catch(Exception ex) { 
-                Console.WriteLine(ex.ToString()); }
-
-
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.ToString());
+            }
             return "UNKNOWN";
         }
 
-        static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
+        static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
         {
-            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
+            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
 
-            RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
-                                                                    PermitLimit = 300,
-                                                                    QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
-                                                                    QueueLimit = 1,
-                                                                    Window = TimeSpan.FromMinutes(1), 
-                                                                    SegmentsPerWindow = 60, 
-                                                                    AutoReplenishment = true} );
+            RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
+            {
+                PermitLimit = 300,
+                QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                QueueLimit = 1,
+                Window = TimeSpan.FromMinutes(1),
+                SegmentsPerWindow = 60,
+                AutoReplenishment = true
+            });
 
             try
             {
                 using (StreamWriter sw = new StreamWriter(outPath, true))
                 {
-                    foreach (var post in posts)
+                    while (posts.Count > 0)
                     {
+                        var post = posts[0]; // Process the first post in the list
                         string status;
 
                         using RateLimitLease lease = limiter.AttemptAcquire(1);
                         if (lease.IsAcquired)
                         {
                             Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
-                            // Thread.Sleep(1000);
                             status = await GrabNotes(post);
                         }
                         else
@@ -455,10 +581,18 @@ namespace URLNotesGrabberCORE
                             Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                             return;
                         }
-                        if(status == "Success")
+
+                        if (status == "Success")
+                        {
                             DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
+                        }
                         else
+                        {
                             Console.WriteLine("GrabNotes Result: " + status);
+                        }
+
+                        // Re-fetch the updated list after processing the current post
+                        posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
                     }
                 }
             }
@@ -469,38 +603,26 @@ namespace URLNotesGrabberCORE
         }
 
 
-        static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
+        static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "")
         {
             // Get all directories in the current directory and sort them alphabetically
             var directories = Directory.GetDirectories(path);
             Array.Sort(directories, StringComparer.InvariantCulture);
 
-            //using (StreamWriter sw = new StreamWriter(outPath, true))
-            //{
-            //sw.WriteLine("=====" + path);
-            //}
-
             foreach (var directory in directories)
             {
-
                 Console.WriteLine("Directory: " + directory);
-                TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
+                TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories
             }
+
             try
             {
                 bool headerWasWritten = false;
                 // Process all files in the current directory
                 foreach (var file in Directory.GetFiles(path))
                 {
-                    if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
+                    if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
                     {
-                        //Console.WriteLine($"=====File: {file}");
-
-                        //using (StreamWriter sw = new StreamWriter(outPath, true))
-                        //{
-                        //sw.WriteLine("---" + file);
-                        //}
-
                         try
                         {
                             var urls = new List<string>();
@@ -508,12 +630,11 @@ namespace URLNotesGrabberCORE
 
                             foreach (string line in File.ReadLines(file))
                             {
-                                //if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
-                                if (line.StartsWith(@"Post id:"))
-                                {   //  if there were no files, let's still collect post info
+                                if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
+                                {
                                     if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
                                     {
-                                        if (!reblog.reblogURL.Contains(@"deactivated")
+                                        if (!reblog.reblogURL.Contains("deactivated")
                                             && reblog.reblogURL.Length != 0
                                             && ContainsAny(reblog.reblogURL, contains))
                                         {
@@ -522,119 +643,109 @@ namespace URLNotesGrabberCORE
                                             {
                                                 headerWasWritten = true;
                                             }
-
-                                            //if (reblog.reblogURL.Contains("/blog/private")
-                                            //    || reblog.body.Contains("/blog/private"))
-                                            //{
-                                            //    Console.WriteLine(reblog.postID);
-                                            //    Console.WriteLine(reblog.postURL);
-                                            //    Console.WriteLine(reblog.date);
-                                            //    Console.WriteLine(reblog.body);
-                                            //    Console.WriteLine(reblog.reblogKey);
-                                            //    Console.WriteLine(reblog.reblogURL);
-                                            //    Console.WriteLine(reblog.reblogName);
-                                            //    Console.WriteLine(reblog.downloadedFiles);
-                                            //}
                                             string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
                                             urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
                                             DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                 reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                 reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                 reblog.title, false);
+                                            postsAdded++;
+
+                                            // Output hyperlink and post date
+                                            Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                         }
                                     }
-
                                     reblog = new ReblogRecord();
                                     reblog.postID = line.Substring(9).Trim();
                                 }
-                                if (line.StartsWith(@"Reblog url:"))
+                                if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     //reblog = new ReblogRecord();
 
                                     reblog.reblogURL = line.Substring(12).Trim();
                                 }
-                                if (line.StartsWith(@"Reblog name:"))
+                                if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.reblogName = line.Substring(13).Trim();
                                 }
-                                if (line.StartsWith(@"Downloaded files:"))
+                                if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.downloadedFiles = line.Substring(17).Trim();
                                 }
-                                if (line.StartsWith(@"Reblog key:"))
+                                if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.reblogKey = line.Substring(11).Trim();
                                 }
-                                if (line.StartsWith(@"Date:"))
+                                if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.date = line.Substring(6).Trim();
                                 }
-                                if (line.StartsWith(@"Body:"))
+                                if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.body = line.Substring(6).Trim();
                                 }
-                                if (line.StartsWith(@"Post url:"))
+                                if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.postURL = line.Substring(10).Trim();
                                 }
-                                if (line.StartsWith(@"Answer:"))
+                                if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.answer = line.Substring(8).Trim();
                                 }
-                                if (line.StartsWith(@"Audio Caption:"))
+                                if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.audioCaption = line.Substring(15).Trim();
                                 }
-                                if (line.StartsWith(@"Blog Name:"))
+                                if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.blogName = line.Substring(11).Trim();
                                 }
-                                if (line.StartsWith(@"Link:"))
+                                if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.link = line.Substring(6).Trim();
                                 }
-                                if (line.StartsWith(@"Photo Caption:"))
+                                if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.photoCaption = line.Substring(15).Trim();
                                 }
-                                if (line.StartsWith(@"Photo url:"))
+                                if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.photoURL = line.Substring(11).Trim();
                                 }
-                                if (line.StartsWith(@"Question:"))
+                                if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.question = line.Substring(10).Trim();
                                 }
-                                if (line.StartsWith(@"Quote:"))
+                                if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.quote = line.Substring(7).Trim();
                                 }
-                                if (line.StartsWith(@"Slug:"))
+                                if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.slug = line.Substring(6).Trim();
                                 }
-                                if (line.StartsWith(@"Summary:"))
+                                if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.summary = line.Substring(9).Trim();
                                 }
-                                if (line.StartsWith(@"Tags:"))
+                                if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.tags = line.Substring(6).Trim();
                                 }
-                                if (line.StartsWith(@"Title:"))
+                                if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
                                 {
                                     reblog.title = line.Substring(7).Trim();
                                 }
 
                                 if ((reblog.downloadedFiles != "."
-                                        || reblog.reblogURL.Contains("/blog/private")
-                                        || reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
+                                        || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
+                                        || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
                                 {
                                     if ((ContainsAny(reblog.downloadedFiles, contains)
-                                        || reblog.reblogURL.Contains("/blog/private")
-                                        || reblog.body.Contains("/blog/private"))
-                                        // && !ContainsAny(reblog.reblogURL, contains)
-                                        && !reblog.reblogURL.Contains(@"deactivated"))
+                                        || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
+                                        || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
+                                        //&& !reblog.reblogURL.Contains("deactivated")
+                                        || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
                                     {
                                         DirectoryInfo currentDir = new DirectoryInfo(path);
                                         if (!headerWasWritten)
@@ -659,6 +770,10 @@ namespace URLNotesGrabberCORE
                                                                 reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                 reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                 reblog.title, true);
+                                        postsAdded++;
+
+                                        // Output hyperlink and post date
+                                        Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                     }
                                 }
                             }

URLNotesGrabberCORE/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "URLNotesGrabberCORE": {
       "commandName": "Project",
-      "commandLineArgs": "-collect 0"
+      "commandLineArgs": "-collect 1"
     }
   }
 }

URLNotesGrabberCORE/ResponseNotes.cs

@@ -83,4 +83,37 @@ namespace URLNotesGrabberCORE
 
         public string rawJson { get; set; }
     }
+
+    // Classes for Posts API endpoint (for reply_text)
+    public class PostsResponse
+    {
+        public List<Post> posts { get; set; }
+    }
+
+    public class Post
+    {
+        public long id { get; set; }
+        public List<NoteDetail> notes { get; set; }
+    }
+
+    public class NoteDetail
+    {
+        public long timestamp { get; set; }
+        public string type { get; set; }
+        public string blog_name { get; set; }
+        public string reply_text { get; set; }
+    }
+
+    public class PostsRoot
+    {
+        public Meta meta { get; set; }
+        [JsonConverter(typeof(EmptyArrayOrObjectConverter<PostsResponse>))]
+        public PostsResponse response { get; set; }
+
+        public string statusCode { get; set; }
+
+        public int retryInSeconds { get; set; }
+
+        public string rawJson { get; set; }
+    }
 }