jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add incremental refresh + cooldown to -likes mode

Browse Source 
After backfill completes for a blog, -likes can now pick up only newer
likes instead of being a one-shot pull. Tracks a per-blog
liked_timestamp high-water mark and stops the refresh walk once it
crosses the stored mark. A configurable cooldown (LikesRefreshCooldownDays,
default 7) gates which blogs are re-checked on each run. -force bypasses
the cooldown.

The migration adds three columns to Blogs (LikesNewestTimestamp,
LikesLastRefreshed, LikesLastNewCount) and does a one-time reset of all
likes tracking state so the new high-water mark starts from a clean
baseline. Existing ByLikes posts remain; the UNIQUE constraint absorbs
re-inserts during the first re-backfill.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
jim
2026-05-16 14:00:45 -05:00
co-authored by Claude Sonnet 4.6
parent 7c667bd579
commit e27190e4f9
3 changed files with 257 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+95 -15
View File
@@ -29,9 +29,17 @@ namespace URLNotesGrabberCORE
string apiSectionName = "TumblrApi";
bool apiExplicitlySet = false;
string startFromBlogName = string.Empty;
bool forceIgnoreCooldown = false;
List<string> filteredArgs = new List<string>();
for (int i = 0; i < args.Length; i++)
{
if (string.Equals(args[i], "-force", StringComparison.OrdinalIgnoreCase) ||
string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
{
forceIgnoreCooldown = true;
continue;
}
if (string.Equals(args[i], "-api3", StringComparison.OrdinalIgnoreCase) ||
string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
{
@@ -159,7 +167,9 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param");
Console.WriteLine("-likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
Console.WriteLine("-force\t (with -likes) Ignore cooldown and refresh every fully-backfilled blog");
Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
@@ -291,7 +301,8 @@ namespace URLNotesGrabberCORE
case "-likes":
string likeBlog = args.Length > 1 ? args[1] : null;
CollectLikes(likeBlog, contains).GetAwaiter().GetResult();
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
break;
case "-urldump":
@@ -681,14 +692,14 @@ namespace URLNotesGrabberCORE
}
}
static async Task CollectLikes(string specificBlog, List<string> contains)
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
{
try
{
DataAccess.EnsureBlogsLikesColumnsExist();
Console.WriteLine("Starting collection of likes...");
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog);
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
if (blogsToProcess.Count == 0)
{
@@ -696,7 +707,9 @@ namespace URLNotesGrabberCORE
return;
}
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes.");
int backfillCount = blogsToProcess.Count(b => b.Item2 == 0);
int refreshCount = blogsToProcess.Count(b => b.Item2 == 1);
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh).");
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{
@@ -713,13 +726,24 @@ namespace URLNotesGrabberCORE
string blogName = blogInfo.Item1;
int likesPulled = blogInfo.Item2;
long cursor = blogInfo.Item3;
long storedNewestTs = blogInfo.Item4;
bool isRefresh = likesPulled == 1;
long parsedForBlog = 0;
long matchedForBlog = 0;
int likedCountForBlog = 0;
long observedMaxLikedTs = storedNewestTs;
int newInsertedInRefresh = 0;
Console.WriteLine($"Processing likes for blog: {blogName} | Cursor: {cursor}");
// Refresh always starts from the top (newest) and walks backward until it crosses
// the stored high-water mark. Backfill resumes from its last persisted cursor.
if (isRefresh) cursor = 0;
string mode = isRefresh ? "REFRESH" : "BACKFILL";
Console.WriteLine($"Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
bool hasMoreLikes = true;
bool isFirstPage = true;
while (hasMoreLikes)
{
@@ -748,14 +772,23 @@ namespace URLNotesGrabberCORE
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
{
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
if (isRefresh)
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
else
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
break;
}
if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
{
Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); // Done parsing
if (isRefresh)
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
else
{
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
}
hasMoreLikes = false;
break;
}
@@ -765,6 +798,8 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}");
bool crossedHighWaterMark = false;
foreach (var post in response.response.liked_posts)
{
parsedForBlog++;
@@ -772,10 +807,24 @@ namespace URLNotesGrabberCORE
long postID = 0;
try { postID = Convert.ToInt64(post.id); } catch { continue; }
// liked_timestamp is when the user liked the post (matches the `before` cursor semantics).
// It's the only reliable field for the refresh stop condition.
long likedTs = 0;
try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { }
if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs)
{
Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh.");
crossedHighWaterMark = true;
break;
}
if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs;
string authorBlog = post.blog_name?.ToString() ?? ".";
string postURL = post.post_url?.ToString() ?? ".";
string date = post.date?.ToString() ?? ".";
// Legacy format fields
string body = post.body?.ToString() ?? ".";
string summary = post.summary?.ToString() ?? ".";
@@ -829,7 +878,7 @@ namespace URLNotesGrabberCORE
// Only insert if any of the data contains strings from ContainsList
bool shouldInsert = false;
string matchedFieldName = string.Empty;
// Let's check a few fields that usually have URLs or relevant info that might match ContainsList
var fieldsToCheck = new (string Name, string Value)[] {
("postURL", postURL),
@@ -863,6 +912,7 @@ namespace URLNotesGrabberCORE
if (shouldInsert)
{
matchedForBlog++;
if (isRefresh) newInsertedInRefresh++;
Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
await Task.Delay(3000);
DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
@@ -874,6 +924,22 @@ namespace URLNotesGrabberCORE
WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
// Branch B: refresh terminates as soon as we crossed the high-water mark.
if (isRefresh && crossedHighWaterMark)
{
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
hasMoreLikes = false;
break;
}
// Branch A: on the very first page, capture & persist the newest liked_timestamp
// so subsequent refresh runs (after backfill completes) have a stopping point.
if (!isRefresh && isFirstPage && observedMaxLikedTs > 0)
{
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
}
isFirstPage = false;
// Determine the next BeforeCursor.
long nextCursor = 0;
if (response.response._links?.next?.query_params != null)
@@ -881,9 +947,12 @@ namespace URLNotesGrabberCORE
long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
}
// Persist cursor progress after every page so resume is always up-to-date
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
if (!isRefresh)
{
// Persist cursor progress after every page so resume is always up-to-date
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
}
if (nextCursor > 0)
{
@@ -893,13 +962,24 @@ namespace URLNotesGrabberCORE
else
{
Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist); // Mark as completely pulled
if (isRefresh)
{
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
}
else
{
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist);
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
}
hasMoreLikes = false;
}
await Task.Delay(1000); // 1-second delay between pages
}
if (isRefresh)
Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}");
WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
}