10 Commits
Author SHA1 Message Date
jim c09c180f1a Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 14:18:14 -05:00
jimandClaude Opus 5.5 a3a54a12fe feat(likes): skip blogs that have never left a note
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 14:18:14 -05:00
jim ed84d6fec0 Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 13:56:28 -05:00
jimandClaude Opus 5.5 a6522779eb feat(likes): order blogs by earliest note, then name
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:56:27 -05:00
jim c5bdd93708 Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 13:38:34 -05:00
jimandClaude Opus 5.5 d16ab14691 feat(likes): show blog progress count and percentage
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:38:33 -05:00
jimandClaude Opus 5.5 3448d75030 fix: trim ContainsList entries and drop empty ones
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:37:14 -05:00
jim 445b52f0c8 Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 13:31:38 -05:00
jimandClaude Opus 5.5 c5676eb084 fix(likes): select all active blogs, not only self-rebloggers
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:31:28 -05:00
jimandClaude 7557c9a168 feat(likes): accept multiple blog names with --likes
Change --likes from accepting a single blog name to accepting a list.
Passes all non-flag args after --likes as blog names. Database stores
bare blog names, so users pass just the name without .tumblr.com.

Co-Authored-By: Claude <[email protected]>
2026-10-19 13:41:05 -05:00
2 changed files with 41 additions and 46 deletions

No files matched your search

+25 -38
View File
@@ -1341,7 +1341,7 @@ namespace URLNotesGrabberCORE
return count; return count;
} }
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null) public static List<Tuple<string, int, long, long>> GetBlogsForLikes(List<string> specificBlogs = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -1352,61 +1352,48 @@ namespace URLNotesGrabberCORE
connection.Open(); connection.Open();
string sql; string sql;
// The two Notes branches below join on Blogs.BlogId, which is NULL for the 168k if (specificBlogs != null && specificBlogs.Count > 0)
// registry rows that have never appeared in a note. The inner join drops them,
// which is correct here -- both branches already require a note to exist -- but
// it is the wrong shape for anything that lists the registry.
if (!string.IsNullOrEmpty(specificBlog))
{ {
// Specific blog: always process, bypass cooldown // Specific blogs: always process, bypass cooldown
sql = @"SELECT BlogName, var placeholders = string.Join(", ", specificBlogs.Select((_, i) => $"@blog{i}"));
sql = $@"SELECT BlogName,
COALESCE(LikesPulled, 0), COALESCE(LikesPulled, 0),
COALESCE(LikesCursor, 0), COALESCE(LikesCursor, 0),
COALESCE(LikesNewestTimestamp, 0) COALESCE(LikesNewestTimestamp, 0)
FROM Blogs FROM Blogs
WHERE BlogName = @blog WHERE BlogName IN ({placeholders})
AND IsActive = 1"; AND IsActive = 1";
} }
else if (ignoreCooldown)
{
// All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
WHERE N.TimeStamp >= 1535778000
AND N.RootBlogId = B.BlogId
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
GROUP BY B.BlogName
ORDER BY MIN(N.TimeStamp);";
}
else else
{ {
// Backfill-pending OR refresh-due (past cooldown window) // Only blogs that have left a note; least recently refreshed first, then earliest first note
string dueFilter = ignoreCooldown ? string.Empty : @"
AND (
COALESCE(B.LikesPulled, 0) = 0
OR COALESCE(B.LikesLastRefreshed, 0)
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
)";
sql = @"SELECT B.BlogName, sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0), COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0), COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0) COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId INNER JOIN (SELECT N.NoteBlogId, MIN(N.TimeStamp) AS FirstNoteTs
WHERE N.TimeStamp >= 1535778000 FROM Notes N" + WhereIsActive("Notes", "N", DBPath) + @"
AND N.RootBlogId = B.BlogId GROUP BY N.NoteBlogId) F ON F.NoteBlogId = B.BlogId
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @" WHERE B.IsActive = 1" + dueFilter + @"
AND ( ORDER BY COALESCE(B.LikesLastRefreshed, 0),
B.LikesPulled = 0 F.FirstNoteTs,
OR COALESCE(B.LikesLastRefreshed, 0) B.BlogName;";
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
)
GROUP BY B.BlogName
ORDER BY MIN(N.TimeStamp);";
} }
using (SQLiteCommand command = new SQLiteCommand(sql, connection)) using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
if (!string.IsNullOrEmpty(specificBlog)) if (specificBlogs != null && specificBlogs.Count > 0)
command.Parameters.AddWithValue("@blog", specificBlog); {
for (int i = 0; i < specificBlogs.Count; i++)
command.Parameters.AddWithValue($"@blog{i}", specificBlogs[i]);
}
else if (!ignoreCooldown) else if (!ignoreCooldown)
command.Parameters.AddWithValue("@cooldownDays", cooldownDays); command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
+16 -8
View File
@@ -170,9 +170,10 @@ namespace URLNotesGrabberCORE
} }
string? containsListSetting = settings.GetValue<string>("ContainsList"); string? containsListSetting = settings.GetValue<string>("ContainsList");
if (string.IsNullOrEmpty(containsListSetting)) List<string> contains = (containsListSetting ?? string.Empty)
.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToList();
if (contains.Count == 0)
throw new InvalidOperationException("ContainsList is not configured in appsettings.json"); throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
List<string> contains = containsListSetting.Split(',').ToList();
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false); bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
@@ -415,9 +416,15 @@ namespace URLNotesGrabberCORE
break; break;
case "--likes": case "--likes":
string likeBlog = args.Length > 1 ? args[1] : null; List<string> likeBlogs = new List<string>();
for (int j = 1; j < args.Length; j++)
{
string arg = args[j];
if (arg.StartsWith("--")) break; // stop at next flag
likeBlogs.Add(arg);
}
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7); int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult(); CollectLikes(likeBlogs, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
break; break;
case "--urldump": case "--urldump":
@@ -509,7 +516,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database"); Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run."); Console.WriteLine("--likes [blog1 [blog2 ...]]\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. One or more blog names restricts to those blogs.");
Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown"); Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
@@ -913,14 +920,14 @@ namespace URLNotesGrabberCORE
} }
} }
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false) static async Task CollectLikes(List<string> specificBlogs, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
{ {
try try
{ {
DataAccess.EnsureBlogsLikesColumnsExist(); DataAccess.EnsureBlogsLikesColumnsExist();
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})"); Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown); var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlogs, cooldownDays, ignoreCooldown);
if (blogsToProcess.Count == 0) if (blogsToProcess.Count == 0)
{ {
@@ -965,7 +972,8 @@ namespace URLNotesGrabberCORE
if (isRefresh) cursor = 0; if (isRefresh) cursor = 0;
string mode = isRefresh ? "REFRESH" : "BACKFILL"; string mode = isRefresh ? "REFRESH" : "BACKFILL";
Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}"); int pct = blogsToProcess.Count > 0 ? (((i + 1) * 100) / blogsToProcess.Count) : 0;
Console.WriteLine($"[{i + 1}/{blogsToProcess.Count} ({pct}%) | {remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
bool hasMoreLikes = true; bool hasMoreLikes = true;
bool isFirstPage = true; bool isFirstPage = true;