Add support for fetching and storing reply text

- Add database migration and update logic for replyText column in Notes
- Implement batch collection of missing reply text via Tumblr API
- Add new API integration to fetch reply text for replies
- Enhance DataAccess with methods to query/update replyText
- Update CLI: -replies now collects and stores reply text
- Improve logging (archive logs/), error handling, and output
- Remove TL.db from source control

URLNotesGrabberCORE/DataAccess.cs

@@ -81,6 +81,57 @@ namespace URLNotesGrabberCORE
             return dateTime;
         }
 
+        public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                // Check if replyText column exists
+                string checkSql = "PRAGMA table_info(Notes);";
+                using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
+                {
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        bool columnExists = false;
+                        while (reader.Read())
+                        {
+                            string columnName = reader.GetString(1);
+                            if (columnName.Equals("replyText", StringComparison.OrdinalIgnoreCase))
+                            {
+                                columnExists = true;
+                                break;
+                            }
+                        }
+
+                        if (!columnExists)
+                        {
+                            // Add the column if it doesn't exist
+                            connection.Close();
+                            connection.Open();
+
+                            string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';";
+                            using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection))
+                            {
+                                addCommand.ExecuteNonQuery();
+                                Console.WriteLine("[Migration] Added replyText column to Notes table");
+                            }
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
         #region Adds
         public static void AddBlog(string blogName, string DBPath = @"TL.db")
         {
@@ -93,10 +144,14 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "INSERT OR IGNORE INTO Blogs (BlogName) values('" + blogName + "')";
+                // Insert BlogName and DateAdded (current UTC datetime)
-                SQLiteCommand command = new SQLiteCommand(sql, connection);
+                string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded) VALUES (@BlogName, @DateAdded)";
-
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
-                command.ExecuteNonQuery();
+                {
+                    command.Parameters.AddWithValue("@BlogName", blogName);
+                    command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    command.ExecuteNonQuery();
+                }
                 // Console.WriteLine("+ " + blogName);
             }
             catch (Exception ex)
@@ -226,7 +281,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)";
+                string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
@@ -234,6 +289,7 @@ namespace URLNotesGrabberCORE
                     command.Parameters.AddWithValue("@PostID", postID);
                     command.Parameters.AddWithValue("@TimeStamp", timestamp);
                     command.Parameters.AddWithValue("@Type", type ?? string.Empty);
+                    command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
 
                     command.ExecuteNonQuery();
                 }
@@ -272,9 +328,9 @@ namespace URLNotesGrabberCORE
                 connection.Open();
 
                 string sql = "SELECT " +
-                             "  Posts.BlogName, " + Environment.NewLine +
+                             "  MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
                              "  Posts.PostID, " + Environment.NewLine +
-                             "  Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
+                             " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
                              "  Posts.NotesGatheredDatetime, " + Environment.NewLine +
                              "  CNT.CNT " + Environment.NewLine +
                              "FROM " + Environment.NewLine +
@@ -283,7 +339,7 @@ namespace URLNotesGrabberCORE
                              "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
                              "      LEFT OUTER JOIN " + Environment.NewLine +
                              "   (      select BlogName, count(PostID) as CNT from Posts group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
-                             "WHERE NotFound = 0 " + Environment.NewLine;
+                             "WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine;
 
                 if (withoutNotesOnly)
                     sql += " and HasNotesGathered = 0 " + Environment.NewLine;
@@ -298,12 +354,12 @@ namespace URLNotesGrabberCORE
                 sql += "GROUP BY " + Environment.NewLine +
                              "   Posts.BlogName, Posts.PostID " + Environment.NewLine +
                              "ORDER BY " + Environment.NewLine +
-                             " notesgathereddatetime,  Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
+                             " notesgathereddatetime,  Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine;
 
-                Console.WriteLine(withoutNotesOnly);
+                //Console.WriteLine(withoutNotesOnly);
-                Console.WriteLine(sql);
+                //Console.WriteLine(sql);
-                Console.Write(">"); //Console.ReadKey();
+                //Console.Write(">"); //Console.ReadKey();
-                Thread.Sleep(250);
+                //Thread.Sleep(250);
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
@@ -382,6 +438,107 @@ namespace URLNotesGrabberCORE
             return posts;
         }
 
+        public static List<Tuple<string, long>> GetRepliesWithMissingText(string DBPath = @"TL.db", int limit = 50)
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
+
+            try
+            {
+                connection.Open();
+
+                string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
+                             MAX(Notes.timestamp) as LatestTimestamp
+                             FROM Notes 
+                             WHERE Notes.type = 'reply' 
+                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                             GROUP BY Notes.RootBlogName, Notes.PostID
+                             ORDER BY LatestTimestamp ASC
+                             LIMIT @limit";
+
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@limit", limit);
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            Tuple<string, long> post = default;
+                            string blog = null;
+                            long id = 0;
+
+                            blog = reader.GetString(reader.GetOrdinal("blogName"));
+                            id = reader.GetInt64(reader.GetOrdinal("PostID"));
+
+                            post = new Tuple<string, long>(blog, id);
+                            posts.Add(post);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return posts;
+        }
+
+        public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string DBPath = @"TL.db", int limit = 1)
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
+
+            try
+            {
+                connection.Open();
+
+                string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
+                             MAX(Notes.timestamp) as LatestTimestamp
+                             FROM Notes 
+                             WHERE Notes.type = 'reply' 
+                             AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
+                             GROUP BY Notes.RootBlogName, Notes.PostID
+                             ORDER BY LatestTimestamp ASC
+                             LIMIT @limit";
+
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@limit", limit);
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            Tuple<string, long, long> post = default;
+                            string blog = null;
+                            long id = 0;
+                            long timestamp = 0;
+
+                            blog = reader.GetString(reader.GetOrdinal("blogName"));
+                            id = reader.GetInt64(reader.GetOrdinal("PostID"));
+                            timestamp = reader.GetInt64(reader.GetOrdinal("LatestTimestamp"));
+                            timestamp++;
+
+                            post = new Tuple<string, long, long>(blog, id, timestamp);
+                            posts.Add(post);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return posts;
+        }
+
         public static int GetAPICount(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -517,7 +674,8 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
+                //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
+                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE PostID = @PostID";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
@@ -760,6 +918,87 @@ namespace URLNotesGrabberCORE
 
             return APICount;
         }
+
+        public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
+                string sql = "UPDATE Notes SET replyText = @replyText WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@replyText", replyText ?? ".");
+                    command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
+                    command.Parameters.AddWithValue("@PostID", postID);
+                    command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
+                    command.Parameters.AddWithValue("@TimeStamp", timestamp);
+                    int rowsAffected = command.ExecuteNonQuery();
+                    
+                    if (rowsAffected == 0)
+                    {
+                        Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}");
+                        Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
+                        Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
+                    }
+                    else
+                    {
+                        Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}");
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
+                Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+        public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@replyText", replyText ?? ".");
+                    command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
+                    command.Parameters.AddWithValue("@PostID", postID);
+                    int rowsAffected = command.ExecuteNonQuery();
+                    
+                    if (rowsAffected == 0)
+                    {
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
+                    }
+                    else
+                    {
+                        Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'");
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
+                Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
         #endregion Updates
     }
 
@@ -790,24 +1029,27 @@ namespace URLNotesGrabberCORE
 
         public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
         {
-            var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]";
+            var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all";
             URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
             if (!string.IsNullOrEmpty(timestamp))
             {
                 URL += "&before_timestamp=" + timestamp;
                 await Task.Delay(100);
             }
-            var client = new RestClient(URL);
-            var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
-                                                            consumerSecret: ConsumerSecret,
-                                                            token: OAuthToken,
-                                                            tokenSecret: OAuthTokenSecret,
-                                                            OAuthSignatureMethod.HmacSha1
-                                                            );
             
-            client.Authenticator = oAuth1;
+            // Create a new RestClient for each request to ensure fresh OAuth signatures
-            var request = new RestRequest(URL, Method.Get);
+            using (var client = new RestClient(URL))
-            var response = await client.ExecuteAsync(request);
+            {
+                var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
+                                                                consumerSecret: ConsumerSecret,
+                                                                token: OAuthToken,
+                                                                tokenSecret: OAuthTokenSecret,
+                                                                OAuthSignatureMethod.HmacSha1
+                                                                );
+
+                client.Authenticator = oAuth1;
+                var request = new RestRequest(URL, Method.Get);
+                var response = await client.ExecuteAsync(request);
 
             var myJsonResponse = response.Content ?? string.Empty;
             Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
@@ -962,11 +1204,75 @@ namespace URLNotesGrabberCORE
                                 myDeserializedClass.statusCode = "TooManyRequests";
                             }
                         }
-                    }
+                }
                 }
             }
 
             return myDeserializedClass;
+            }
+        }
+
+        public static async Task<PostsRoot> GrabPostWithReplies(string blog, long postID, long timestamp)
+        {
+            var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}&notes_info=true&before_timestamp={timestamp}";
+
+            // Create a new RestClient for each request to ensure fresh OAuth signatures
+            using (var client = new RestClient(URL))
+            {
+                var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
+                                                                consumerSecret: ConsumerSecret,
+                                                                token: OAuthToken,
+                                                                tokenSecret: OAuthTokenSecret,
+                                                                OAuthSignatureMethod.HmacSha1
+                                                                );
+
+                client.Authenticator = oAuth1;
+                var request = new RestRequest(URL, Method.Get);
+                var response = await client.ExecuteAsync(request);
+
+                var myJsonResponse = response.Content ?? string.Empty;
+                Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
+                var myDeserializedClass = new PostsRoot();
+
+                // Check if response is successful and contains JSON
+                if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
+                {
+                    string? statusStr = null;
+                    try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
+                    Console.WriteLine($"[Reply API] {statusStr}\t{response?.StatusDescription}");
+                    if (!string.IsNullOrEmpty(statusStr))
+                        myDeserializedClass.statusCode = statusStr;
+                    return myDeserializedClass;
+                }
+
+                // Check if response is HTML (error page) instead of JSON
+                if (myJsonResponse.TrimStart().StartsWith("<"))
+                {
+                    Console.WriteLine($"[Reply API] Received HTML response (likely error page): {response?.StatusCode}");
+                    string? statusStr = null;
+                    try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
+                    if (!string.IsNullOrEmpty(statusStr))
+                        myDeserializedClass.statusCode = statusStr;
+                    return myDeserializedClass;
+                }
+
+                try
+                {
+                    var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
+                    if (deserializedResult != null)
+                    {
+                        myDeserializedClass = deserializedResult;
+                        myDeserializedClass.rawJson = myJsonResponse;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
+                    Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
+                }
+
+                return myDeserializedClass;
+            }
         }
     }
 }

URLNotesGrabberCORE/Program.cs

@@ -26,9 +26,18 @@ namespace URLNotesGrabberCORE
             if (File.Exists(logPath))
             {
                 string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
-                string newPath = $"console_output_{timestamp}.log";
+                string archiveDirectory = "logs";
+
+                // Ensure the archive directory exists
+                if (!Directory.Exists(archiveDirectory))
+                {
+                    Directory.CreateDirectory(archiveDirectory);
+                }
+
+                string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
                 File.Move(logPath, newPath);
             }
+
             StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
             DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
             Console.SetOut(dualLogger);
@@ -37,7 +46,9 @@ namespace URLNotesGrabberCORE
 
             if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
             {
-                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
+                int postsAdded = 0;
+                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded);
+                Console.WriteLine($"Total posts added: {postsAdded}");
             }
             else
             {
@@ -62,96 +73,19 @@ namespace URLNotesGrabberCORE
 
                         Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
 
-                        Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
+                        Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
 
                         break;
+                    
                     case "-parse":
                         string blogNameToParse = args[1];
-
+                        int postsAdded = 0;
-                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
+                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse);
-
+                        Console.WriteLine($"Total posts added: {postsAdded}");
                         break;
                     
                     case "-test":
-                        #region Manual test 
+                        Console.WriteLine("Test command not implemented");
-
-                        string blogName = args[1];
-                        long postID = long.Parse(args[2]);
-
-                        var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
-                        List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
-
-                        // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound
-                        if (response?.meta != null && response.meta.status == 404)
-                        {
-                            Console.WriteLine($"ERROR: Post not found - {blogName}/{postID} (meta.status=404)");
-                            DataAccess.UpdatePostMarkNotFound(blogName, postID);
-                            break;
-                        }
-
-                        // Check for API errors
-                        if (response.statusCode == "NotFound")
-                        {
-                            Console.WriteLine($"ERROR: Post not found - {blogName}/{postID}");
-                            break;
-                        }
-                        if (response.statusCode == "TooManyRequests")
-                        {
-                            Console.WriteLine($"ERROR: Rate limited - retry in {response.retryInSeconds} seconds");
-                            break;
-                        }
-                        if (!string.IsNullOrEmpty(response.statusCode))
-                        {
-                            Console.WriteLine($"ERROR: API returned status code: {response.statusCode}");
-                            break;
-                        }
-
-                        // Check for null response
-                        if (response.response == null)
-                        {
-                            Console.WriteLine("ERROR: API response is null");
-                            break;
-                        }
-
-                        // Check for null notes
-                        if (response.response.notes == null)
-                        {
-                            Console.WriteLine("ERROR: Notes collection is null (post may have 0 notes)");
-                            break;
-                        }
-
-                        Console.WriteLine($"Processing {response.response.notes.Count} notes...");
-                        foreach (var note in response.response.notes)
-                        {
-                            note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
-                            //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                            if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                                break;
-                        }
-
-                        while (response.response._links != null)
-                        {
-                            response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
-
-                            if (response.response?.notes == null)
-                            {
-                                Console.WriteLine("ERROR: Notes collection became null during pagination");
-                                break;
-                            }
-
-                            Console.WriteLine($"Processing {response.response.notes.Count} more notes...");
-                            foreach (var note in response.response.notes)
-                            {
-                                note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
-                                //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                                if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                                    break;
-                            }
-                        }
-
-                        Console.WriteLine("Test completed successfully");
-                        #endregion
-
                         break;
 
                     case "-post":
@@ -168,7 +102,7 @@ namespace URLNotesGrabberCORE
 
                     case "-collect": //collect notes from all posts
                         bool withoutNotesOnly = true;
-                        DateTime? beforeDate = null;
+                        DateTime? beforeDate = DateTime.Now;
 
                         if (args.Length < 2)
                         {
@@ -247,10 +181,8 @@ namespace URLNotesGrabberCORE
                         WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
                         break;
 
-                    case "-replies": //colection posts with replies
+                    case "-replies": //update reply text
-                        try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
+                        CollectMissingReplyText().GetAwaiter().GetResult();
-                        WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
-
                         break;
 
                     default:
@@ -351,16 +283,181 @@ namespace URLNotesGrabberCORE
             return false;
         }
 
+        static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
+        {
+            try
+            {
+                Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
+                // Add 2-second delay before API call to avoid server-side rate limiting
+                await Task.Delay(2000);
+                var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp);
+
+                if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
+                {
+                    Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
+                    Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
+                    Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
+                    if (postsResponse?.response != null)
+                    {
+                        Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
+                    }
+                    if (!string.IsNullOrEmpty(postsResponse?.rawJson))
+                    {
+                        Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
+                    }
+                    
+                    // Mark all replies for this post with '?' to indicate API returned no posts
+                    Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
+                    DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
+                    
+                    return;
+                }
+
+                // Get the first (and should be only) post
+                var post = postsResponse.response.posts.FirstOrDefault();
+                if (post?.notes == null)
+                {
+                    Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
+                    return;
+                }
+
+                Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");
+
+                // Update each reply with its text
+                int replyCount = 0;
+                int skippedCount = 0;
+                foreach (var note in post.notes)
+                {
+                    if (note.type == "reply")
+                    {
+                        if (!string.IsNullOrEmpty(note.reply_text))
+                        {
+                            DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
+                            replyCount++;
+                            
+                            // Output the reply text being stored
+                            string displayText = note.reply_text.Length > 100 
+                                ? note.reply_text.Substring(0, 100) + "..." 
+                                : note.reply_text;
+                            Console.WriteLine($"  [{note.blog_name}] {displayText}");
+                        }
+                        else
+                        {
+                            skippedCount++;
+                            Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
+                        }
+                    }
+                }
+
+                if (replyCount > 0)
+                {
+                    Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
+                }
+                else
+                {
+                    Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
+                }
+
+                // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
+                Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
+                //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
+                //if (cleanupCount > 0)
+                //{
+                //    Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
+                //}
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}");
+                Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
+            }
+        }
+
+        static async Task CollectMissingReplyText()
+        {
+            try
+            {
+                // Ensure the replyText column exists
+                DataAccess.EnsureReplyTextColumnExists();
+
+                Console.WriteLine("Starting collection of missing reply text...");
+                Console.WriteLine("Processing 10 posts at a time.");
+
+                int batchSize = 1;  // Process 10 posts per batch
+                int totalProcessedCount = 0;
+
+                RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
+                {
+                    PermitLimit = 300,
+                    QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                    QueueLimit = 1,
+                    Window = TimeSpan.FromMinutes(1),
+                    SegmentsPerWindow = 60,
+                    AutoReplenishment = true
+                });
+
+                while (true && totalProcessedCount < batchSize)
+                {
+                    var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
+                    
+                    if (postsWithFilledReplies.Count == 0)
+                    {
+                        Console.WriteLine("No more posts with filled reply text found. Collection complete.");
+                        break;
+                    }
+
+                    Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
+                    int batchProcessedCount = 0;
+
+                    foreach (var post in postsWithFilledReplies)
+                    {
+                        using RateLimitLease lease = limiter.AttemptAcquire(1);
+                        if (lease.IsAcquired)
+                        {
+                            Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
+                            await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
+                            batchProcessedCount++;
+                            totalProcessedCount++;
+                        
+                            // Add 1 second delay between attempts
+                            await Task.Delay(1000);
+                        }
+                        else
+                        {
+                            Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
+                            Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
+                            return;
+                        }
+                    }
+
+                    Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
+                }
+
+                Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
+                Console.WriteLine(ex.ToString());
+            }
+        }
+
         static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
         {
             try
             {
                 int APICount = DataAccess.GetAPICount();
+                Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
+                Thread.Sleep(3000);
 
-                Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
+                var allNotes = new List<dynamic>();
-                var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
+                int page = 1;
+                string beforeTimestamp = post.Item3.ToString();
+                bool hasReplies = false;
+                const int maxPages = 500;
+                var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
 
-                // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return
+                // Handle 404 and error codes
                 if (response?.meta != null && response.meta.status == 404)
                 {
                     Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)");
@@ -368,8 +465,11 @@ namespace URLNotesGrabberCORE
                     Thread.Sleep(1000);
                     return "NotFound";
                 }
-                List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
+                if (response == null)
-
+                {
+                    Console.WriteLine("##### Response is null - API Failure? ###");
+                    return "FAILURE";
+                }
                 if (response.statusCode == "NotFound")
                 {
                     Thread.Sleep(1000);
@@ -386,50 +486,56 @@ namespace URLNotesGrabberCORE
                     return response.statusCode;
                 }
 
-                if (response.response != null && response.response.notes != null)
+                // Pagination loop
+                while (true)
                 {
-                    Console.WriteLine("Notes\t" + response.response.notes.Count);
+                    int noteCount = response?.response?.notes?.Count ?? 0;
-                    foreach (var note in response.response.notes)
+                    Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");
-                    {
-                        note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
-                        //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                        if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                            break;
-                    }
-                }
-                else if (response.response == null)
-                {
-                    Console.WriteLine("##### Response Null - API Failure? ###");
-                    Console.WriteLine($"Raw JSON: {response.rawJson}");
-                    return "FAILURE";
-                }
-                else if (response.response.notes == null)
-                {
-                    Console.WriteLine("##### Notes Null - WHY? ###");
-                    return "FAILURE";
-                }
-                else
-                {
-                    while (response.response != null
-                             && response.response._links != null
-                             && long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
-                    {
-                        response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
-                        if (response.response.notes is null)
-                        {
-                            Console.WriteLine("response.response.notes == null");
-                            return "NULL NOTES";
-                        }
 
-                        Console.WriteLine("Notes\t" + response.response.notes.Count);
+                    if (response?.response == null)
-                        foreach (var note in response.response.notes)
+                    {
-                        {
+                        Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
-                            note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
+                        return "FAILURE";
-                            //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
-                            if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
-                                break;
-                        }
                     }
+                    if (response.response.notes == null)
+                    {
+                        Console.WriteLine("##### Notes Null - WHY? ###");
+                        return "FAILURE";
+                    }
+
+                    // Accumulate notes
+                    allNotes.AddRange(response.response.notes);
+
+                    // Pagination: check for next
+                    var nextLink = response.response._links?.next;
+                    if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
+                    {
+                        Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
+                        break;
+                    }
+
+                    beforeTimestamp = nextLink.query_params.before_timestamp;
+                    page++;
+                    if (page > maxPages)
+                    {
+                        Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination.");
+                        break;
+                    }
+
+                    // Fetch next page
+                    response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
+                }
+
+                Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
+
+                // Process all accumulated notes
+                foreach (var note in allNotes)
+                {
+                    note.reblog_parent_blog_name = post.Item1;
+                    note.post_id = post.Item2.ToString();
+                    if (note.type == "reply")
+                        hasReplies = true;
+                    DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
                 }
 
                 return "Success";
@@ -438,8 +544,6 @@ namespace URLNotesGrabberCORE
             {
                 Console.WriteLine(ex.ToString());
             }
-
-
             return "UNKNOWN";
         }
 
@@ -461,15 +565,15 @@ namespace URLNotesGrabberCORE
             {
                 using (StreamWriter sw = new StreamWriter(outPath, true))
                 {
-                    foreach (var post in posts)
+                    while (posts.Count > 0)
                     {
+                        var post = posts[0]; // Process the first post in the list
                         string status;
 
                         using RateLimitLease lease = limiter.AttemptAcquire(1);
                         if (lease.IsAcquired)
                         {
                             Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
-                            // Thread.Sleep(1000);
                             status = await GrabNotes(post);
                         }
                         else
@@ -477,10 +581,18 @@ namespace URLNotesGrabberCORE
                             Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                             return;
                         }
+
                         if (status == "Success")
+                        {
                             DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
+                        }
                         else
+                        {
                             Console.WriteLine("GrabNotes Result: " + status);
+                        }
+
+                        // Re-fetch the updated list after processing the current post
+                        posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
                     }
                 }
             }
@@ -491,27 +603,18 @@ namespace URLNotesGrabberCORE
         }
 
 
-        static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
+        static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "")
         {
-            //if(!ContainsAny(path, contains))
-            //{
-            //    return;
-            //}
             // Get all directories in the current directory and sort them alphabetically
             var directories = Directory.GetDirectories(path);
             Array.Sort(directories, StringComparer.InvariantCulture);
 
-            //using (StreamWriter sw = new StreamWriter(outPath, true))
-            //{
-            //sw.WriteLine("=====" + path);
-            //}
-
             foreach (var directory in directories)
             {
-
                 Console.WriteLine("Directory: " + directory);
-                TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
+                TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories
             }
+
             try
             {
                 bool headerWasWritten = false;
@@ -520,13 +623,6 @@ namespace URLNotesGrabberCORE
                 {
                     if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
                     {
-                        //Console.WriteLine($"=====File: {file}");
-
-                        //using (StreamWriter sw = new StreamWriter(outPath, true))
-                        //{
-                        //sw.WriteLine("---" + file);
-                        //}
-
                         try
                         {
                             var urls = new List<string>();
@@ -534,12 +630,11 @@ namespace URLNotesGrabberCORE
 
                             foreach (string line in File.ReadLines(file))
                             {
-                                //if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
+                                if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
-                                if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase))
+                                {
-                                {   //  if there were no files, let's still collect post info
                                     if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
                                     {
-                                        if (!reblog.reblogURL.Contains(@"deactivated")
+                                        if (!reblog.reblogURL.Contains("deactivated")
                                             && reblog.reblogURL.Length != 0
                                             && ContainsAny(reblog.reblogURL, contains))
                                         {
@@ -548,28 +643,18 @@ namespace URLNotesGrabberCORE
                                             {
                                                 headerWasWritten = true;
                                             }
-
-                                            //if (reblog.reblogURL.Contains("/blog/private")
-                                            //    || reblog.body.Contains("/blog/private"))
-                                            //{
-                                            //    Console.WriteLine(reblog.postID);
-                                            //    Console.WriteLine(reblog.postURL);
-                                            //    Console.WriteLine(reblog.date);
-                                            //    Console.WriteLine(reblog.body);
-                                            //    Console.WriteLine(reblog.reblogKey);
-                                            //    Console.WriteLine(reblog.reblogURL);
-                                            //    Console.WriteLine(reblog.reblogName);
-                                            //    Console.WriteLine(reblog.downloadedFiles);
-                                            //}
                                             string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
                                             urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
                                             DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
                                                                 reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                 reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                 reblog.title, false);
+                                            postsAdded++;
+
+                                            // Output hyperlink and post date
+                                            Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                         }
                                     }
-
                                     reblog = new ReblogRecord();
                                     reblog.postID = line.Substring(9).Trim();
                                 }
@@ -659,8 +744,7 @@ namespace URLNotesGrabberCORE
                                     if ((ContainsAny(reblog.downloadedFiles, contains)
                                         || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
                                         || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
-                                        // && !ContainsAny(reblog.reblogURL, contains)
+                                        //&& !reblog.reblogURL.Contains("deactivated")
-                                        && !reblog.reblogURL.Contains(@"deactivated")
                                         || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
                                     {
                                         DirectoryInfo currentDir = new DirectoryInfo(path);
@@ -686,6 +770,10 @@ namespace URLNotesGrabberCORE
                                                                 reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
                                                                 reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
                                                                 reblog.title, true);
+                                        postsAdded++;
+
+                                        // Output hyperlink and post date
+                                        Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
                                     }
                                 }
                             }

URLNotesGrabberCORE/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "URLNotesGrabberCORE": {
       "commandName": "Project",
-      "commandLineArgs": "-test zomb-eh 802499389473079296"
+      "commandLineArgs": "-collect 1"
     }
   }
 }

URLNotesGrabberCORE/ResponseNotes.cs

@@ -83,4 +83,37 @@ namespace URLNotesGrabberCORE
 
         public string rawJson { get; set; }
     }
+
+    // Classes for Posts API endpoint (for reply_text)
+    public class PostsResponse
+    {
+        public List<Post> posts { get; set; }
+    }
+
+    public class Post
+    {
+        public long id { get; set; }
+        public List<NoteDetail> notes { get; set; }
+    }
+
+    public class NoteDetail
+    {
+        public long timestamp { get; set; }
+        public string type { get; set; }
+        public string blog_name { get; set; }
+        public string reply_text { get; set; }
+    }
+
+    public class PostsRoot
+    {
+        public Meta meta { get; set; }
+        [JsonConverter(typeof(EmptyArrayOrObjectConverter<PostsResponse>))]
+        public PostsResponse response { get; set; }
+
+        public string statusCode { get; set; }
+
+        public int retryInSeconds { get; set; }
+
+        public string rawJson { get; set; }
+    }
 }