diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index e10e9f8..1f7f201 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -314,6 +314,9 @@ namespace URLNotesGrabberCORE string checkSql = "PRAGMA table_info(Blogs);"; bool likesPulledExists = false; bool likesCursorExists = false; + bool likesNewestTimestampExists = false; + bool likesLastRefreshedExists = false; + bool likesLastNewCountExists = false; using (SQLiteCommand command = new SQLiteCommand(checkSql, connection)) { @@ -324,6 +327,9 @@ namespace URLNotesGrabberCORE string columnName = reader.GetString(1); if (columnName.Equals("LikesPulled", StringComparison.OrdinalIgnoreCase)) likesPulledExists = true; if (columnName.Equals("LikesCursor", StringComparison.OrdinalIgnoreCase)) likesCursorExists = true; + if (columnName.Equals("LikesNewestTimestamp", StringComparison.OrdinalIgnoreCase)) likesNewestTimestampExists = true; + if (columnName.Equals("LikesLastRefreshed", StringComparison.OrdinalIgnoreCase)) likesLastRefreshedExists = true; + if (columnName.Equals("LikesLastNewCount", StringComparison.OrdinalIgnoreCase)) likesLastNewCountExists = true; } } } @@ -341,6 +347,45 @@ namespace URLNotesGrabberCORE using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery(); Console.WriteLine("[Migration] Added LikesCursor column to Blogs table"); } + + if (!likesNewestTimestampExists) + { + string addCol = "ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;"; + using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery(); + Console.WriteLine("[Migration] Added LikesNewestTimestamp column to Blogs table"); + } + + if (!likesLastRefreshedExists) + { + string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;"; + using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery(); + Console.WriteLine("[Migration] Added LikesLastRefreshed column to Blogs table"); + } + + if (!likesLastNewCountExists) + { + string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;"; + using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery(); + Console.WriteLine("[Migration] Added LikesLastNewCount column to Blogs table"); + } + + // One-time reset: if any of the new high-water-mark columns were just added, + // wipe all likes tracking state so the system starts from a known-good baseline. + // Existing Posts rows with ByLikes=1 remain — UNIQUE(BlogName, PostID) absorbs re-inserts. + if (!likesNewestTimestampExists || !likesLastRefreshedExists || !likesLastNewCountExists) + { + string resetSql = @"UPDATE Blogs + SET LikesPulled = 0, + LikesCursor = 0, + LikesNewestTimestamp = 0, + LikesLastRefreshed = 0, + LikesLastNewCount = 0;"; + using (SQLiteCommand cmd = new SQLiteCommand(resetSql, connection)) + { + int affected = cmd.ExecuteNonQuery(); + Console.WriteLine($"[Migration] Reset likes tracking on {affected} blog rows for clean baseline"); + } + } } catch (Exception ex) { @@ -1012,34 +1057,76 @@ namespace URLNotesGrabberCORE return count; } - public static List> GetBlogsForLikes(string specificBlog = null, string? DBPath = null) + public static List> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); - List> blogs = new List>(); + List> blogs = new List>(); try { connection.Open(); string sql; if (!string.IsNullOrEmpty(specificBlog)) - sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog"; + { + // Specific blog: always process, bypass cooldown + sql = @"SELECT BlogName, + COALESCE(LikesPulled, 0), + COALESCE(LikesCursor, 0), + COALESCE(LikesNewestTimestamp, 0) + FROM Blogs + WHERE BlogName = @blog"; + } + else if (ignoreCooldown) + { + // All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown + sql = @"SELECT B.BlogName, + COALESCE(B.LikesPulled, 0), + COALESCE(B.LikesCursor, 0), + COALESCE(B.LikesNewestTimestamp, 0) + FROM Blogs B + INNER JOIN Notes N ON N.NoteBlogName = B.BlogName + WHERE N.TimeStamp >= 1535778000 + AND N.rootBlogName = B.BlogName + GROUP BY B.BlogName + ORDER BY MIN(N.Timestamp);"; + } else - sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);"; + { + // Backfill-pending OR refresh-due (past cooldown window) + sql = @"SELECT B.BlogName, + COALESCE(B.LikesPulled, 0), + COALESCE(B.LikesCursor, 0), + COALESCE(B.LikesNewestTimestamp, 0) + FROM Blogs B + INNER JOIN Notes N ON N.NoteBlogName = B.BlogName + WHERE N.TimeStamp >= 1535778000 + AND N.rootBlogName = B.BlogName + AND ( + B.LikesPulled = 0 + OR COALESCE(B.LikesLastRefreshed, 0) + < (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400)) + ) + GROUP BY B.BlogName + ORDER BY MIN(N.Timestamp);"; + } using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { if (!string.IsNullOrEmpty(specificBlog)) command.Parameters.AddWithValue("@blog", specificBlog); + else if (!ignoreCooldown) + command.Parameters.AddWithValue("@cooldownDays", cooldownDays); using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { - blogs.Add(new Tuple( + blogs.Add(new Tuple( reader.GetString(0), reader.GetInt32(1), - reader.GetInt64(2) + reader.GetInt64(2), + reader.GetInt64(3) )); } } @@ -1500,6 +1587,73 @@ namespace URLNotesGrabberCORE } } + // Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we + // capture the newest liked_timestamp on the first page so subsequent refresh runs have a + // stopping point. MAX(...) protects against out-of-order updates. + public static void UpdateBlogLikesNewestTimestamp(string blogName, long newestTimestamp, string? DBPath = null) + { + if (newestTimestamp <= 0) return; + DBPath ??= GetDefaultDbPath(); + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + try + { + connection.Open(); + string sql = @"UPDATE Blogs + SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest), + DateModified = @modified + WHERE BlogName = @name"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@newest", newestTimestamp); + command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.Parameters.AddWithValue("@name", blogName); + command.ExecuteNonQuery(); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}"); + } + finally + { + connection.Close(); + } + } + + // Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the + // last-refreshed time so cooldown takes effect, and records how many new likes were found. + public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null) + { + DBPath ??= GetDefaultDbPath(); + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + try + { + connection.Open(); + string sql = @"UPDATE Blogs + SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest), + LikesLastRefreshed = CAST(strftime('%s','now') AS INTEGER), + LikesLastNewCount = @count, + DateModified = @modified + WHERE BlogName = @name"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@newest", newestTimestamp); + command.Parameters.AddWithValue("@count", newCount); + command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.Parameters.AddWithValue("@name", blogName); + command.ExecuteNonQuery(); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}"); + } + finally + { + connection.Close(); + } + } + public static int UpdateAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 3507fcc..7ef231c 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -29,9 +29,17 @@ namespace URLNotesGrabberCORE string apiSectionName = "TumblrApi"; bool apiExplicitlySet = false; string startFromBlogName = string.Empty; + bool forceIgnoreCooldown = false; List filteredArgs = new List(); for (int i = 0; i < args.Length; i++) { + if (string.Equals(args[i], "-force", StringComparison.OrdinalIgnoreCase) || + string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase)) + { + forceIgnoreCooldown = true; + continue; + } + if (string.Equals(args[i], "-api3", StringComparison.OrdinalIgnoreCase) || string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase)) { @@ -159,7 +167,9 @@ namespace URLNotesGrabberCORE Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database"); - Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param"); + Console.WriteLine("-likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run."); + + Console.WriteLine("-force\t (with -likes) Ignore cooldown and refresh every fully-backfilled blog"); Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file"); @@ -291,7 +301,8 @@ namespace URLNotesGrabberCORE case "-likes": string likeBlog = args.Length > 1 ? args[1] : null; - CollectLikes(likeBlog, contains).GetAwaiter().GetResult(); + int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7); + CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult(); break; case "-urldump": @@ -681,14 +692,14 @@ namespace URLNotesGrabberCORE } } - static async Task CollectLikes(string specificBlog, List contains) + static async Task CollectLikes(string specificBlog, List contains, int cooldownDays = 7, bool ignoreCooldown = false) { try { DataAccess.EnsureBlogsLikesColumnsExist(); - Console.WriteLine("Starting collection of likes..."); - var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog); + Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})"); + var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown); if (blogsToProcess.Count == 0) { @@ -696,7 +707,9 @@ namespace URLNotesGrabberCORE return; } - Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes."); + int backfillCount = blogsToProcess.Count(b => b.Item2 == 0); + int refreshCount = blogsToProcess.Count(b => b.Item2 == 1); + Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh)."); RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { @@ -713,13 +726,24 @@ namespace URLNotesGrabberCORE string blogName = blogInfo.Item1; int likesPulled = blogInfo.Item2; long cursor = blogInfo.Item3; + long storedNewestTs = blogInfo.Item4; + + bool isRefresh = likesPulled == 1; long parsedForBlog = 0; long matchedForBlog = 0; int likedCountForBlog = 0; + long observedMaxLikedTs = storedNewestTs; + int newInsertedInRefresh = 0; - Console.WriteLine($"Processing likes for blog: {blogName} | Cursor: {cursor}"); + // Refresh always starts from the top (newest) and walks backward until it crosses + // the stored high-water mark. Backfill resumes from its last persisted cursor. + if (isRefresh) cursor = 0; + + string mode = isRefresh ? "REFRESH" : "BACKFILL"; + Console.WriteLine($"Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}"); bool hasMoreLikes = true; + bool isFirstPage = true; while (hasMoreLikes) { @@ -748,14 +772,23 @@ namespace URLNotesGrabberCORE if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404)) { Console.WriteLine($"API returned 404 Not Found for {blogName} Likes"); - DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); + if (isRefresh) + DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh); + else + DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); break; } if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0) { Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete."); - DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); // Done parsing + if (isRefresh) + DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh); + else + { + DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); + DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs); + } hasMoreLikes = false; break; } @@ -765,6 +798,8 @@ namespace URLNotesGrabberCORE Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}"); + bool crossedHighWaterMark = false; + foreach (var post in response.response.liked_posts) { parsedForBlog++; @@ -772,10 +807,24 @@ namespace URLNotesGrabberCORE long postID = 0; try { postID = Convert.ToInt64(post.id); } catch { continue; } + // liked_timestamp is when the user liked the post (matches the `before` cursor semantics). + // It's the only reliable field for the refresh stop condition. + long likedTs = 0; + try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { } + + if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs) + { + Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh."); + crossedHighWaterMark = true; + break; + } + + if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs; + string authorBlog = post.blog_name?.ToString() ?? "."; string postURL = post.post_url?.ToString() ?? "."; string date = post.date?.ToString() ?? "."; - + // Legacy format fields string body = post.body?.ToString() ?? "."; string summary = post.summary?.ToString() ?? "."; @@ -829,7 +878,7 @@ namespace URLNotesGrabberCORE // Only insert if any of the data contains strings from ContainsList bool shouldInsert = false; string matchedFieldName = string.Empty; - + // Let's check a few fields that usually have URLs or relevant info that might match ContainsList var fieldsToCheck = new (string Name, string Value)[] { ("postURL", postURL), @@ -863,6 +912,7 @@ namespace URLNotesGrabberCORE if (shouldInsert) { matchedForBlog++; + if (isRefresh) newInsertedInRefresh++; Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}"); await Task.Delay(3000); DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey, @@ -874,6 +924,22 @@ namespace URLNotesGrabberCORE WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog); + // Branch B: refresh terminates as soon as we crossed the high-water mark. + if (isRefresh && crossedHighWaterMark) + { + DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh); + hasMoreLikes = false; + break; + } + + // Branch A: on the very first page, capture & persist the newest liked_timestamp + // so subsequent refresh runs (after backfill completes) have a stopping point. + if (!isRefresh && isFirstPage && observedMaxLikedTs > 0) + { + DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs); + } + isFirstPage = false; + // Determine the next BeforeCursor. long nextCursor = 0; if (response.response._links?.next?.query_params != null) @@ -881,9 +947,12 @@ namespace URLNotesGrabberCORE long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor); } - // Persist cursor progress after every page so resume is always up-to-date - long cursorToPersist = nextCursor > 0 ? nextCursor : cursor; - DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist); + if (!isRefresh) + { + // Persist cursor progress after every page so resume is always up-to-date + long cursorToPersist = nextCursor > 0 ? nextCursor : cursor; + DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist); + } if (nextCursor > 0) { @@ -893,13 +962,24 @@ namespace URLNotesGrabberCORE else { Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}."); - DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist); // Mark as completely pulled + if (isRefresh) + { + DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh); + } + else + { + long cursorToPersist = nextCursor > 0 ? nextCursor : cursor; + DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist); + DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs); + } hasMoreLikes = false; } await Task.Delay(1000); // 1-second delay between pages } + if (isRefresh) + Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}"); WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog); } diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index 413f809..f3bccea 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -11,7 +11,8 @@ "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot", "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218", "EnableFileLogging": false, - "LogTraversalRecordImports": false + "LogTraversalRecordImports": false, + "LikesRefreshCooldownDays": 7 }, "TumblrApi": { "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",