jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add incremental refresh + cooldown to -likes mode

Browse Source 
After backfill completes for a blog, -likes can now pick up only newer
likes instead of being a one-shot pull. Tracks a per-blog
liked_timestamp high-water mark and stops the refresh walk once it
crosses the stored mark. A configurable cooldown (LikesRefreshCooldownDays,
default 7) gates which blogs are re-checked on each run. -force bypasses
the cooldown.

The migration adds three columns to Blogs (LikesNewestTimestamp,
LikesLastRefreshed, LikesLastNewCount) and does a one-time reset of all
likes tracking state so the new high-water mark starts from a clean
baseline. Existing ByLikes posts remain; the UNIQUE constraint absorbs
re-inserts during the first re-backfill.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
jim
2026-05-16 14:00:45 -05:00
co-authored by Claude Sonnet 4.6
parent 7c667bd579
commit e27190e4f9
3 changed files with 257 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+160 -6
View File
@@ -314,6 +314,9 @@ namespace URLNotesGrabberCORE
string checkSql = "PRAGMA table_info(Blogs);";
bool likesPulledExists = false;
bool likesCursorExists = false;
bool likesNewestTimestampExists = false;
bool likesLastRefreshedExists = false;
bool likesLastNewCountExists = false;
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
{
@@ -324,6 +327,9 @@ namespace URLNotesGrabberCORE
string columnName = reader.GetString(1);
if (columnName.Equals("LikesPulled", StringComparison.OrdinalIgnoreCase)) likesPulledExists = true;
if (columnName.Equals("LikesCursor", StringComparison.OrdinalIgnoreCase)) likesCursorExists = true;
if (columnName.Equals("LikesNewestTimestamp", StringComparison.OrdinalIgnoreCase)) likesNewestTimestampExists = true;
if (columnName.Equals("LikesLastRefreshed", StringComparison.OrdinalIgnoreCase)) likesLastRefreshedExists = true;
if (columnName.Equals("LikesLastNewCount", StringComparison.OrdinalIgnoreCase)) likesLastNewCountExists = true;
}
}
}
@@ -341,6 +347,45 @@ namespace URLNotesGrabberCORE
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesCursor column to Blogs table");
}
if (!likesNewestTimestampExists)
{
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;";
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesNewestTimestamp column to Blogs table");
}
if (!likesLastRefreshedExists)
{
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;";
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesLastRefreshed column to Blogs table");
}
if (!likesLastNewCountExists)
{
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;";
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesLastNewCount column to Blogs table");
}
// One-time reset: if any of the new high-water-mark columns were just added,
// wipe all likes tracking state so the system starts from a known-good baseline.
// Existing Posts rows with ByLikes=1 remain — UNIQUE(BlogName, PostID) absorbs re-inserts.
if (!likesNewestTimestampExists || !likesLastRefreshedExists || !likesLastNewCountExists)
{
string resetSql = @"UPDATE Blogs
SET LikesPulled = 0,
LikesCursor = 0,
LikesNewestTimestamp = 0,
LikesLastRefreshed = 0,
LikesLastNewCount = 0;";
using (SQLiteCommand cmd = new SQLiteCommand(resetSql, connection))
{
int affected = cmd.ExecuteNonQuery();
Console.WriteLine($"[Migration] Reset likes tracking on {affected} blog rows for clean baseline");
}
}
}
catch (Exception ex)
{
@@ -1012,34 +1057,76 @@ namespace URLNotesGrabberCORE
return count;
}
public static List<Tuple<string, int, long>> GetBlogsForLikes(string specificBlog = null, string? DBPath = null)
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long>> blogs = new List<Tuple<string, int, long>>();
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
try
{
connection.Open();
string sql;
if (!string.IsNullOrEmpty(specificBlog))
sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
{
// Specific blog: always process, bypass cooldown
sql = @"SELECT BlogName,
COALESCE(LikesPulled, 0),
COALESCE(LikesCursor, 0),
COALESCE(LikesNewestTimestamp, 0)
FROM Blogs
WHERE BlogName = @blog";
}
else if (ignoreCooldown)
{
// All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
GROUP BY B.BlogName
ORDER BY MIN(N.Timestamp);";
}
else
sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
{
// Backfill-pending OR refresh-due (past cooldown window)
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
AND (
B.LikesPulled = 0
OR COALESCE(B.LikesLastRefreshed, 0)
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
)
GROUP BY B.BlogName
ORDER BY MIN(N.Timestamp);";
}
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
if (!string.IsNullOrEmpty(specificBlog))
command.Parameters.AddWithValue("@blog", specificBlog);
else if (!ignoreCooldown)
command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
blogs.Add(new Tuple<string, int, long>(
blogs.Add(new Tuple<string, int, long, long>(
reader.GetString(0),
reader.GetInt32(1),
reader.GetInt64(2)
reader.GetInt64(2),
reader.GetInt64(3)
));
}
}
@@ -1500,6 +1587,73 @@ namespace URLNotesGrabberCORE
}
}
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
// capture the newest liked_timestamp on the first page so subsequent refresh runs have a
// stopping point. MAX(...) protects against out-of-order updates.
public static void UpdateBlogLikesNewestTimestamp(string blogName, long newestTimestamp, string? DBPath = null)
{
if (newestTimestamp <= 0) return;
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = @"UPDATE Blogs
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
DateModified = @modified
WHERE BlogName = @name";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@newest", newestTimestamp);
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@name", blogName);
command.ExecuteNonQuery();
}
}
catch (Exception ex)
{
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
}
finally
{
connection.Close();
}
}
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
// last-refreshed time so cooldown takes effect, and records how many new likes were found.
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = @"UPDATE Blogs
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
LikesLastRefreshed = CAST(strftime('%s','now') AS INTEGER),
LikesLastNewCount = @count,
DateModified = @modified
WHERE BlogName = @name";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@newest", newestTimestamp);
command.Parameters.AddWithValue("@count", newCount);
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@name", blogName);
command.ExecuteNonQuery();
}
}
catch (Exception ex)
{
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
}
finally
{
connection.Close();
}
}
public static int UpdateAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();