jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'claude/gifted-dhawan-7e5fc7'

Browse Source
This commit is contained in:
jim
2026-05-16 14:02:07 -05:00
parent 7c667bd579 e27190e4f9
commit 5781e121d2
3 changed files with 257 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+160 -6
View File
@@ -314,6 +314,9 @@ namespace URLNotesGrabberCORE
string checkSql = "PRAGMA table_info(Blogs);";
bool likesPulledExists = false;
bool likesCursorExists = false;
bool likesNewestTimestampExists = false;
bool likesLastRefreshedExists = false;
bool likesLastNewCountExists = false;
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
{
@@ -324,6 +327,9 @@ namespace URLNotesGrabberCORE
string columnName = reader.GetString(1);
if (columnName.Equals("LikesPulled", StringComparison.OrdinalIgnoreCase)) likesPulledExists = true;
if (columnName.Equals("LikesCursor", StringComparison.OrdinalIgnoreCase)) likesCursorExists = true;
if (columnName.Equals("LikesNewestTimestamp", StringComparison.OrdinalIgnoreCase)) likesNewestTimestampExists = true;
if (columnName.Equals("LikesLastRefreshed", StringComparison.OrdinalIgnoreCase)) likesLastRefreshedExists = true;
if (columnName.Equals("LikesLastNewCount", StringComparison.OrdinalIgnoreCase)) likesLastNewCountExists = true;
}
}
}
@@ -341,6 +347,45 @@ namespace URLNotesGrabberCORE
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesCursor column to Blogs table");
}
if (!likesNewestTimestampExists)
{
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;";
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesNewestTimestamp column to Blogs table");
}
if (!likesLastRefreshedExists)
{
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;";
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesLastRefreshed column to Blogs table");
}
if (!likesLastNewCountExists)
{
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;";
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added LikesLastNewCount column to Blogs table");
}
// One-time reset: if any of the new high-water-mark columns were just added,
// wipe all likes tracking state so the system starts from a known-good baseline.
// Existing Posts rows with ByLikes=1 remain — UNIQUE(BlogName, PostID) absorbs re-inserts.
if (!likesNewestTimestampExists || !likesLastRefreshedExists || !likesLastNewCountExists)
{
string resetSql = @"UPDATE Blogs
SET LikesPulled = 0,
LikesCursor = 0,
LikesNewestTimestamp = 0,
LikesLastRefreshed = 0,
LikesLastNewCount = 0;";
using (SQLiteCommand cmd = new SQLiteCommand(resetSql, connection))
{
int affected = cmd.ExecuteNonQuery();
Console.WriteLine($"[Migration] Reset likes tracking on {affected} blog rows for clean baseline");
}
}
}
catch (Exception ex)
{
@@ -1012,34 +1057,76 @@ namespace URLNotesGrabberCORE
return count;
}
public static List<Tuple<string, int, long>> GetBlogsForLikes(string specificBlog = null, string? DBPath = null)
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long>> blogs = new List<Tuple<string, int, long>>();
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
try
{
connection.Open();
string sql;
if (!string.IsNullOrEmpty(specificBlog))
sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
{
// Specific blog: always process, bypass cooldown
sql = @"SELECT BlogName,
COALESCE(LikesPulled, 0),
COALESCE(LikesCursor, 0),
COALESCE(LikesNewestTimestamp, 0)
FROM Blogs
WHERE BlogName = @blog";
}
else if (ignoreCooldown)
{
// All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
GROUP BY B.BlogName
ORDER BY MIN(N.Timestamp);";
}
else
sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
{
// Backfill-pending OR refresh-due (past cooldown window)
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogName = B.BlogName
WHERE N.TimeStamp >= 1535778000
AND N.rootBlogName = B.BlogName
AND (
B.LikesPulled = 0
OR COALESCE(B.LikesLastRefreshed, 0)
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
)
GROUP BY B.BlogName
ORDER BY MIN(N.Timestamp);";
}
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
if (!string.IsNullOrEmpty(specificBlog))
command.Parameters.AddWithValue("@blog", specificBlog);
else if (!ignoreCooldown)
command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
blogs.Add(new Tuple<string, int, long>(
blogs.Add(new Tuple<string, int, long, long>(
reader.GetString(0),
reader.GetInt32(1),
reader.GetInt64(2)
reader.GetInt64(2),
reader.GetInt64(3)
));
}
}
@@ -1500,6 +1587,73 @@ namespace URLNotesGrabberCORE
}
}
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
// capture the newest liked_timestamp on the first page so subsequent refresh runs have a
// stopping point. MAX(...) protects against out-of-order updates.
public static void UpdateBlogLikesNewestTimestamp(string blogName, long newestTimestamp, string? DBPath = null)
{
if (newestTimestamp <= 0) return;
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = @"UPDATE Blogs
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
DateModified = @modified
WHERE BlogName = @name";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@newest", newestTimestamp);
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@name", blogName);
command.ExecuteNonQuery();
}
}
catch (Exception ex)
{
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
}
finally
{
connection.Close();
}
}
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
// last-refreshed time so cooldown takes effect, and records how many new likes were found.
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = @"UPDATE Blogs
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
LikesLastRefreshed = CAST(strftime('%s','now') AS INTEGER),
LikesLastNewCount = @count,
DateModified = @modified
WHERE BlogName = @name";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@newest", newestTimestamp);
command.Parameters.AddWithValue("@count", newCount);
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.Parameters.AddWithValue("@name", blogName);
command.ExecuteNonQuery();
}
}
catch (Exception ex)
{
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
}
finally
{
connection.Close();
}
}
public static int UpdateAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
URLNotesGrabberCORE/Program.cs
+89 -9
View File
@@ -29,9 +29,17 @@ namespace URLNotesGrabberCORE
string apiSectionName = "TumblrApi";
bool apiExplicitlySet = false;
string startFromBlogName = string.Empty;
bool forceIgnoreCooldown = false;
List<string> filteredArgs = new List<string>();
for (int i = 0; i < args.Length; i++)
{
if (string.Equals(args[i], "-force", StringComparison.OrdinalIgnoreCase) ||
string.Equals(args[i], "--force", StringComparison.OrdinalIgnoreCase))
{
forceIgnoreCooldown = true;
continue;
}
if (string.Equals(args[i], "-api3", StringComparison.OrdinalIgnoreCase) ||
string.Equals(args[i], "--api3", StringComparison.OrdinalIgnoreCase))
{
@@ -159,7 +167,9 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param");
Console.WriteLine("-likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
Console.WriteLine("-force\t (with -likes) Ignore cooldown and refresh every fully-backfilled blog");
Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
@@ -291,7 +301,8 @@ namespace URLNotesGrabberCORE
case "-likes":
string likeBlog = args.Length > 1 ? args[1] : null;
CollectLikes(likeBlog, contains).GetAwaiter().GetResult();
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
break;
case "-urldump":
@@ -681,14 +692,14 @@ namespace URLNotesGrabberCORE
}
}
static async Task CollectLikes(string specificBlog, List<string> contains)
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
{
try
{
DataAccess.EnsureBlogsLikesColumnsExist();
Console.WriteLine("Starting collection of likes...");
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog);
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
if (blogsToProcess.Count == 0)
{
@@ -696,7 +707,9 @@ namespace URLNotesGrabberCORE
return;
}
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes.");
int backfillCount = blogsToProcess.Count(b => b.Item2 == 0);
int refreshCount = blogsToProcess.Count(b => b.Item2 == 1);
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes ({backfillCount} backfill, {refreshCount} refresh).");
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{
@@ -713,13 +726,24 @@ namespace URLNotesGrabberCORE
string blogName = blogInfo.Item1;
int likesPulled = blogInfo.Item2;
long cursor = blogInfo.Item3;
long storedNewestTs = blogInfo.Item4;
bool isRefresh = likesPulled == 1;
long parsedForBlog = 0;
long matchedForBlog = 0;
int likedCountForBlog = 0;
long observedMaxLikedTs = storedNewestTs;
int newInsertedInRefresh = 0;
Console.WriteLine($"Processing likes for blog: {blogName} | Cursor: {cursor}");
// Refresh always starts from the top (newest) and walks backward until it crosses
// the stored high-water mark. Backfill resumes from its last persisted cursor.
if (isRefresh) cursor = 0;
string mode = isRefresh ? "REFRESH" : "BACKFILL";
Console.WriteLine($"Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
bool hasMoreLikes = true;
bool isFirstPage = true;
while (hasMoreLikes)
{
@@ -748,6 +772,9 @@ namespace URLNotesGrabberCORE
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
{
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
if (isRefresh)
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
else
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
break;
}
@@ -755,7 +782,13 @@ namespace URLNotesGrabberCORE
if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
{
Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); // Done parsing
if (isRefresh)
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
else
{
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
}
hasMoreLikes = false;
break;
}
@@ -765,6 +798,8 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}");
bool crossedHighWaterMark = false;
foreach (var post in response.response.liked_posts)
{
parsedForBlog++;
@@ -772,6 +807,20 @@ namespace URLNotesGrabberCORE
long postID = 0;
try { postID = Convert.ToInt64(post.id); } catch { continue; }
// liked_timestamp is when the user liked the post (matches the `before` cursor semantics).
// It's the only reliable field for the refresh stop condition.
long likedTs = 0;
try { likedTs = Convert.ToInt64(post.liked_timestamp); } catch { }
if (isRefresh && likedTs > 0 && storedNewestTs > 0 && likedTs <= storedNewestTs)
{
Console.WriteLine($"[Likes] Reached high-water mark for {blogName} at liked_timestamp={likedTs} (<= stored {storedNewestTs}). Stopping refresh.");
crossedHighWaterMark = true;
break;
}
if (likedTs > observedMaxLikedTs) observedMaxLikedTs = likedTs;
string authorBlog = post.blog_name?.ToString() ?? ".";
string postURL = post.post_url?.ToString() ?? ".";
string date = post.date?.ToString() ?? ".";
@@ -863,6 +912,7 @@ namespace URLNotesGrabberCORE
if (shouldInsert)
{
matchedForBlog++;
if (isRefresh) newInsertedInRefresh++;
Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
await Task.Delay(3000);
DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
@@ -874,6 +924,22 @@ namespace URLNotesGrabberCORE
WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
// Branch B: refresh terminates as soon as we crossed the high-water mark.
if (isRefresh && crossedHighWaterMark)
{
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
hasMoreLikes = false;
break;
}
// Branch A: on the very first page, capture & persist the newest liked_timestamp
// so subsequent refresh runs (after backfill completes) have a stopping point.
if (!isRefresh && isFirstPage && observedMaxLikedTs > 0)
{
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
}
isFirstPage = false;
// Determine the next BeforeCursor.
long nextCursor = 0;
if (response.response._links?.next?.query_params != null)
@@ -881,9 +947,12 @@ namespace URLNotesGrabberCORE
long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
}
if (!isRefresh)
{
// Persist cursor progress after every page so resume is always up-to-date
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
}
if (nextCursor > 0)
{
@@ -893,13 +962,24 @@ namespace URLNotesGrabberCORE
else
{
Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist); // Mark as completely pulled
if (isRefresh)
{
DataAccess.UpdateBlogLikesRefreshStatus(blogName, observedMaxLikedTs, newInsertedInRefresh);
}
else
{
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist);
DataAccess.UpdateBlogLikesNewestTimestamp(blogName, observedMaxLikedTs);
}
hasMoreLikes = false;
}
await Task.Delay(1000); // 1-second delay between pages
}
if (isRefresh)
Console.WriteLine($"[Likes] {blogName} refresh complete | New inserted: {newInsertedInRefresh} | New HighWaterMark: {observedMaxLikedTs}");
WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
}
URLNotesGrabberCORE/appsettings.json
+2 -1
View File
@@ -11,7 +11,8 @@
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot",
"PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218",
"EnableFileLogging": false,
"LogTraversalRecordImports": false
"LogTraversalRecordImports": false,
"LikesRefreshCooldownDays": 7
},
"TumblrApi": {
"ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",