Merge branch 'claude/gifted-dhawan-7e5fc7'

URLNotesGrabberCORE/DataAccess.cs

@@ -314,6 +314,9 @@ namespace URLNotesGrabberCORE
                 string checkSql = "PRAGMA table_info(Blogs);";
                 bool likesPulledExists = false;
                 bool likesCursorExists = false;
+                bool likesNewestTimestampExists = false;
+                bool likesLastRefreshedExists = false;
+                bool likesLastNewCountExists = false;
 
                 using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
                 {
@@ -324,6 +327,9 @@ namespace URLNotesGrabberCORE
                             string columnName = reader.GetString(1);
                             if (columnName.Equals("LikesPulled", StringComparison.OrdinalIgnoreCase)) likesPulledExists = true;
                             if (columnName.Equals("LikesCursor", StringComparison.OrdinalIgnoreCase)) likesCursorExists = true;
+                            if (columnName.Equals("LikesNewestTimestamp", StringComparison.OrdinalIgnoreCase)) likesNewestTimestampExists = true;
+                            if (columnName.Equals("LikesLastRefreshed", StringComparison.OrdinalIgnoreCase)) likesLastRefreshedExists = true;
+                            if (columnName.Equals("LikesLastNewCount", StringComparison.OrdinalIgnoreCase)) likesLastNewCountExists = true;
                         }
                     }
                 }
@@ -341,6 +347,45 @@ namespace URLNotesGrabberCORE
                     using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
                     Console.WriteLine("[Migration] Added LikesCursor column to Blogs table");
                 }
+
+                if (!likesNewestTimestampExists)
+                {
+                    string addCol = "ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;";
+                    using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
+                    Console.WriteLine("[Migration] Added LikesNewestTimestamp column to Blogs table");
+                }
+
+                if (!likesLastRefreshedExists)
+                {
+                    string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;";
+                    using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
+                    Console.WriteLine("[Migration] Added LikesLastRefreshed column to Blogs table");
+                }
+
+                if (!likesLastNewCountExists)
+                {
+                    string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;";
+                    using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
+                    Console.WriteLine("[Migration] Added LikesLastNewCount column to Blogs table");
+                }
+
+                // One-time reset: if any of the new high-water-mark columns were just added,
+                // wipe all likes tracking state so the system starts from a known-good baseline.
+                // Existing Posts rows with ByLikes=1 remain — UNIQUE(BlogName, PostID) absorbs re-inserts.
+                if (!likesNewestTimestampExists || !likesLastRefreshedExists || !likesLastNewCountExists)
+                {
+                    string resetSql = @"UPDATE Blogs
+                                           SET LikesPulled = 0,
+                                               LikesCursor = 0,
+                                               LikesNewestTimestamp = 0,
+                                               LikesLastRefreshed = 0,
+                                               LikesLastNewCount = 0;";
+                    using (SQLiteCommand cmd = new SQLiteCommand(resetSql, connection))
+                    {
+                        int affected = cmd.ExecuteNonQuery();
+                        Console.WriteLine($"[Migration] Reset likes tracking on {affected} blog rows for clean baseline");
+                    }
+                }
             }
             catch (Exception ex)
             {
@@ -1012,34 +1057,76 @@ namespace URLNotesGrabberCORE
             return count;
         }
 
-        public static List<Tuple<string, int, long>> GetBlogsForLikes(string specificBlog = null, string? DBPath = null)
+        public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
-            List<Tuple<string, int, long>> blogs = new List<Tuple<string, int, long>>();
+            List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
 
             try
             {
                 connection.Open();
                 string sql;
                 if (!string.IsNullOrEmpty(specificBlog))
-                    sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
+                {
+                    // Specific blog: always process, bypass cooldown
+                    sql = @"SELECT BlogName,
+                                   COALESCE(LikesPulled, 0),
+                                   COALESCE(LikesCursor, 0),
+                                   COALESCE(LikesNewestTimestamp, 0)
+                              FROM Blogs
+                             WHERE BlogName = @blog";
+                }
+                else if (ignoreCooldown)
+                {
+                    // All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
+                    sql = @"SELECT B.BlogName,
+                                   COALESCE(B.LikesPulled, 0),
+                                   COALESCE(B.LikesCursor, 0),
+                                   COALESCE(B.LikesNewestTimestamp, 0)
+                              FROM Blogs B
+                              INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
+                             WHERE N.TimeStamp >= 1535778000
+                               AND N.rootBlogName = B.BlogName
+                             GROUP BY B.BlogName
+                             ORDER BY MIN(N.Timestamp);";
+                }
                 else
-                    sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0)  FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
+                {
+                    // Backfill-pending OR refresh-due (past cooldown window)
+                    sql = @"SELECT B.BlogName,
+                                   COALESCE(B.LikesPulled, 0),
+                                   COALESCE(B.LikesCursor, 0),
+                                   COALESCE(B.LikesNewestTimestamp, 0)
+                              FROM Blogs B
+                              INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
+                             WHERE N.TimeStamp >= 1535778000
+                               AND N.rootBlogName = B.BlogName
+                               AND (
+                                    B.LikesPulled = 0
+                                 OR COALESCE(B.LikesLastRefreshed, 0)
+                                    < (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
+                               )
+                             GROUP BY B.BlogName
+                             ORDER BY MIN(N.Timestamp);";
+                }
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     if (!string.IsNullOrEmpty(specificBlog))
                         command.Parameters.AddWithValue("@blog", specificBlog);
+                    else if (!ignoreCooldown)
+                        command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
 
                     using (SQLiteDataReader reader = command.ExecuteReader())
                     {
                         while (reader.Read())
                         {
-                            blogs.Add(new Tuple<string, int, long>(
+                            blogs.Add(new Tuple<string, int, long, long>(
                                 reader.GetString(0),
                                 reader.GetInt32(1),
-                                reader.GetInt64(2)
+                                reader.GetInt64(2),
+                                reader.GetInt64(3)
                             ));
                         }
                     }
@@ -1500,6 +1587,73 @@ namespace URLNotesGrabberCORE
             }
         }
 
+        // Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
+        // capture the newest liked_timestamp on the first page so subsequent refresh runs have a
+        // stopping point. MAX(...) protects against out-of-order updates.
+        public static void UpdateBlogLikesNewestTimestamp(string blogName, long newestTimestamp, string? DBPath = null)
+        {
+            if (newestTimestamp <= 0) return;
+            DBPath ??= GetDefaultDbPath();
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            try
+            {
+                connection.Open();
+                string sql = @"UPDATE Blogs
+                                  SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
+                                      DateModified = @modified
+                                WHERE BlogName = @name";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@newest", newestTimestamp);
+                    command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    command.Parameters.AddWithValue("@name", blogName);
+                    command.ExecuteNonQuery();
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+        // Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
+        // last-refreshed time so cooldown takes effect, and records how many new likes were found.
+        public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
+        {
+            DBPath ??= GetDefaultDbPath();
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            try
+            {
+                connection.Open();
+                string sql = @"UPDATE Blogs
+                                  SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
+                                      LikesLastRefreshed = CAST(strftime('%s','now') AS INTEGER),
+                                      LikesLastNewCount = @count,
+                                      DateModified = @modified
+                                WHERE BlogName = @name";
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    command.Parameters.AddWithValue("@newest", newestTimestamp);
+                    command.Parameters.AddWithValue("@count", newCount);
+                    command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    command.Parameters.AddWithValue("@name", blogName);
+                    command.ExecuteNonQuery();
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
         public static int UpdateAPICount(string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();

URLNotesGrabberCORE/Program.cs

@@ -29,9 +29,17 @@ namespace URLNotesGrabberCORE
             string apiSectionName = "TumblrApi";
             bool apiExplicitlySet = false;
             string startFromBlogName = string.Empty;
+            bool forceIgnoreCooldown = false;
             List<string> filteredArgs = new List<string>();
             for (int i = 0; i < args.Length; i++)
             {
+                if (string.Equals(args[i], "-force", StringComparison.OrdinalIgnoreCase) ||
+                    string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
+                {
+                    forceIgnoreCooldown = true;
+                    continue;
+                }
+
                 if (string.Equals(args[i], "-api3", StringComparison.OrdinalIgnoreCase) ||
                     string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
                 {
@@ -159,7 +167,9 @@ namespace URLNotesGrabberCORE
 
                         Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
 
-                        Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param");
+                        Console.WriteLine("-likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
+
+                        Console.WriteLine("-force\t (with -likes) Ignore cooldown and refresh every fully-backfilled blog");
 
                         Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
 
@@ -291,7 +301,8 @@ namespace URLNotesGrabberCORE
 
                     case "-likes":
                         string likeBlog = args.Length > 1 ? args[1] : null;
-                        CollectLikes(likeBlog, contains).GetAwaiter().GetResult();
+                        int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
+                        CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
                         break;
 
                     case "-urldump":
@@ -681,14 +692,14 @@ namespace URLNotesGrabberCORE
             }
         }
 
-        static async Task CollectLikes(string specificBlog, List<string> contains)
+        static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
         {
             try
             {
                 DataAccess.EnsureBlogsLikesColumnsExist();
 
-                Console.WriteLine("Starting collection of likes...");
+                Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
-                var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog);
+                var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
 
                 if (blogsToProcess.Count == 0)
                 {
@@ -696,7 +707,9 @@ namespace URLNotesGrabberCORE
                     return;
                 }
 
-                Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes.");
+                int backfillCount = blogsToProcess.Count(b => b.Item2 == 0);
+                int refreshCount = blogsToProcess.Count(b => b.Item2 == 1);
+                Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh).");
 
                 RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
                 {
@@ -713,13 +726,24 @@ namespace URLNotesGrabberCORE
                     string blogName = blogInfo.Item1;
                     int likesPulled = blogInfo.Item2;
                     long cursor = blogInfo.Item3;
+                    long storedNewestTs = blogInfo.Item4;
+
+                    bool isRefresh = likesPulled == 1;
                     long parsedForBlog = 0;
                     long matchedForBlog = 0;
                     int likedCountForBlog = 0;
+                    long observedMaxLikedTs = storedNewestTs;
+                    int newInsertedInRefresh = 0;
 
-                    Console.WriteLine($"Processing likes for blog: {blogName} | Cursor: {cursor}");
+                    // Refresh always starts from the top (newest) and walks backward until it crosses
+                    // the stored high-water mark. Backfill resumes from its last persisted cursor.
+                    if (isRefresh) cursor = 0;
+
+                    string mode = isRefresh ? "REFRESH" : "BACKFILL";
+                    Console.WriteLine($"Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
 
                     bool hasMoreLikes = true;
+                    bool isFirstPage = true;
 
                     while (hasMoreLikes)
                     {
@@ -748,14 +772,23 @@ namespace URLNotesGrabberCORE
                         if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
                         {
                             Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
-                            DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
+                            if (isRefresh)
+                                DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
+                            else
+                                DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
                             break;
                         }
 
                         if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
                         {
                             Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
-                            DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); // Done parsing
+                            if (isRefresh)
+                                DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
+                            else
+                            {
+                                DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
+                                DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
+                            }
                             hasMoreLikes = false;
                             break;
                         }
@@ -765,6 +798,8 @@ namespace URLNotesGrabberCORE
 
                         Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}");
 
+                        bool crossedHighWaterMark = false;
+
                         foreach (var post in response.response.liked_posts)
                         {
                             parsedForBlog++;
@@ -772,10 +807,24 @@ namespace URLNotesGrabberCORE
                             long postID = 0;
                             try { postID = Convert.ToInt64(post.id); } catch { continue; }
 
+                            // liked_timestamp is when the user liked the post (matches the `before` cursor semantics).
+                            // It's the only reliable field for the refresh stop condition.
+                            long likedTs = 0;
+                            try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { }
+
+                            if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs)
+                            {
+                                Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh.");
+                                crossedHighWaterMark = true;
+                                break;
+                            }
+
+                            if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs;
+
                             string authorBlog = post.blog_name?.ToString() ?? ".";
                             string postURL = post.post_url?.ToString() ?? ".";
                             string date = post.date?.ToString() ?? ".";
-                            
+
                             // Legacy format fields
                             string body = post.body?.ToString() ?? ".";
                             string summary = post.summary?.ToString() ?? ".";
@@ -829,7 +878,7 @@ namespace URLNotesGrabberCORE
                             // Only insert if any of the data contains strings from ContainsList
                             bool shouldInsert = false;
                             string matchedFieldName = string.Empty;
-                            
+
                             // Let's check a few fields that usually have URLs or relevant info that might match ContainsList
                             var fieldsToCheck = new (string Name, string Value)[] {
                                 ("postURL", postURL),
@@ -863,6 +912,7 @@ namespace URLNotesGrabberCORE
                             if (shouldInsert)
                             {
                                 matchedForBlog++;
+                                if (isRefresh) newInsertedInRefresh++;
                                 Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
                                 await Task.Delay(3000);
                                 DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
@@ -874,6 +924,22 @@ namespace URLNotesGrabberCORE
 
                         WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
 
+                        // Branch B: refresh terminates as soon as we crossed the high-water mark.
+                        if (isRefresh && crossedHighWaterMark)
+                        {
+                            DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
+                            hasMoreLikes = false;
+                            break;
+                        }
+
+                        // Branch A: on the very first page, capture & persist the newest liked_timestamp
+                        // so subsequent refresh runs (after backfill completes) have a stopping point.
+                        if (!isRefresh && isFirstPage && observedMaxLikedTs > 0)
+                        {
+                            DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
+                        }
+                        isFirstPage = false;
+
                         // Determine the next BeforeCursor.
                         long nextCursor = 0;
                         if (response.response._links?.next?.query_params != null)
@@ -881,9 +947,12 @@ namespace URLNotesGrabberCORE
                             long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
                         }
 
-                        // Persist cursor progress after every page so resume is always up-to-date
+                        if (!isRefresh)
-                        long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
+                        {
-                        DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
+                            // Persist cursor progress after every page so resume is always up-to-date
+                            long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
+                            DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
+                        }
 
                         if (nextCursor > 0)
                         {
@@ -893,13 +962,24 @@ namespace URLNotesGrabberCORE
                         else
                         {
                             Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
-                            DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist); // Mark as completely pulled
+                            if (isRefresh)
+                            {
+                                DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
+                            }
+                            else
+                            {
+                                long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
+                                DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist);
+                                DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
+                            }
                             hasMoreLikes = false;
                         }
 
                         await Task.Delay(1000); // 1-second delay between pages
                     }
 
+                    if (isRefresh)
+                        Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}");
                     WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
                 }
 

URLNotesGrabberCORE/appsettings.json

@@ -11,7 +11,8 @@
     "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot",
     "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218",
     "EnableFileLogging": false,
-    "LogTraversalRecordImports": false
+    "LogTraversalRecordImports": false,
+    "LikesRefreshCooldownDays": 7
   },
   "TumblrApi": {
     "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",