14 Commits
Author SHA1 Message Date
jim c09c180f1a Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 14:18:14 -05:00
jimandClaude Opus 5.5 a3a54a12fe feat(likes): skip blogs that have never left a note
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 14:18:14 -05:00
jim ed84d6fec0 Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 13:56:28 -05:00
jimandClaude Opus 5.5 a6522779eb feat(likes): order blogs by earliest note, then name
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:56:27 -05:00
jim c5bdd93708 Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 13:38:34 -05:00
jimandClaude Opus 5.5 d16ab14691 feat(likes): show blog progress count and percentage
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:38:33 -05:00
jimandClaude Opus 5.5 3448d75030 fix: trim ContainsList entries and drop empty ones
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:37:14 -05:00
jim 445b52f0c8 Merge branch 'claude/blogs-likes-criteria-5dca9a' into master 2026-10-21 13:31:38 -05:00
jimandClaude Opus 5.5 c5676eb084 fix(likes): select all active blogs, not only self-rebloggers
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 13:31:28 -05:00
jimandClaude 7557c9a168 feat(likes): accept multiple blog names with --likes
Change --likes from accepting a single blog name to accepting a list.
Passes all non-flag args after --likes as blog names. Database stores
bare blog names, so users pass just the name without .tumblr.com.

Co-Authored-By: Claude <[email protected]>
2026-10-19 13:41:05 -05:00
jim ca99586176 Merge branch 'claude/collect-query-count-output-c430c8' into master 2026-10-06 15:29:26 -05:00
jimandClaude Opus 5.5 f40d824a8c feat(collect): print candidate row count after the query
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-06 15:29:18 -05:00
jim bde6506e01 Merge branch 'claude/friendly-bhaskara-b06186' into master 2026-10-02 09:26:26 -05:00
jimandClaude Opus 5.5 06a3f8c142 fix(traverse): import the last no-download record in each file
TraverseDirectory imported a record without downloaded files only when
the next "Post id:" line appeared, so the final record of every export
file was never evaluated and a matching post at the end of a file never
reached Posts.

Move the check into a local function and call it again after the line
loop, so the last record is flushed under exactly the same conditions as
every other one. HasImage values at both call sites are unchanged.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-02 09:26:21 -05:00
2 changed files with 87 additions and 76 deletions

No files matched your search

+33 -38
View File
@@ -1055,6 +1055,7 @@ namespace URLNotesGrabberCORE
if (toDate.HasValue)
command.Parameters.AddWithValue("@toDate", toDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
int rowCount = 0;
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
@@ -1114,6 +1115,8 @@ namespace URLNotesGrabberCORE
throw;
}
rowCount++;
// Exclude if postID is in the exclusion list
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
continue;
@@ -1122,6 +1125,11 @@ namespace URLNotesGrabberCORE
posts.Add(post);
}
}
int excluded = rowCount - posts.Count;
Console.WriteLine(excluded > 0
? $" COUNT: {rowCount} ({excluded} excluded, {posts.Count} to collect)"
: $" COUNT: {rowCount}");
}
}
catch (Exception ex)
@@ -1333,7 +1341,7 @@ namespace URLNotesGrabberCORE
return count;
}
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(List<string> specificBlogs = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -1344,61 +1352,48 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql;
// The two Notes branches below join on Blogs.BlogId, which is NULL for the 168k
// registry rows that have never appeared in a note. The inner join drops them,
// which is correct here -- both branches already require a note to exist -- but
// it is the wrong shape for anything that lists the registry.
if (!string.IsNullOrEmpty(specificBlog))
if (specificBlogs != null && specificBlogs.Count > 0)
{
// Specific blog: always process, bypass cooldown
sql = @"SELECT BlogName,
// Specific blogs: always process, bypass cooldown
var placeholders = string.Join(", ", specificBlogs.Select((_, i) => $"@blog{i}"));
sql = $@"SELECT BlogName,
COALESCE(LikesPulled, 0),
COALESCE(LikesCursor, 0),
COALESCE(LikesNewestTimestamp, 0)
FROM Blogs
WHERE BlogName = @blog
WHERE BlogName IN ({placeholders})
AND IsActive = 1";
}
else if (ignoreCooldown)
{
// All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
WHERE N.TimeStamp >= 1535778000
AND N.RootBlogId = B.BlogId
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
GROUP BY B.BlogName
ORDER BY MIN(N.TimeStamp);";
}
else
{
// Backfill-pending OR refresh-due (past cooldown window)
// Only blogs that have left a note; least recently refreshed first, then earliest first note
string dueFilter = ignoreCooldown ? string.Empty : @"
AND (
COALESCE(B.LikesPulled, 0) = 0
OR COALESCE(B.LikesLastRefreshed, 0)
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
)";
sql = @"SELECT B.BlogName,
COALESCE(B.LikesPulled, 0),
COALESCE(B.LikesCursor, 0),
COALESCE(B.LikesNewestTimestamp, 0)
FROM Blogs B
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
WHERE N.TimeStamp >= 1535778000
AND N.RootBlogId = B.BlogId
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
AND (
B.LikesPulled = 0
OR COALESCE(B.LikesLastRefreshed, 0)
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
)
GROUP BY B.BlogName
ORDER BY MIN(N.TimeStamp);";
INNER JOIN (SELECT N.NoteBlogId, MIN(N.TimeStamp) AS FirstNoteTs
FROM Notes N" + WhereIsActive("Notes", "N", DBPath) + @"
GROUP BY N.NoteBlogId) F ON F.NoteBlogId = B.BlogId
WHERE B.IsActive = 1" + dueFilter + @"
ORDER BY COALESCE(B.LikesLastRefreshed, 0),
F.FirstNoteTs,
B.BlogName;";
}
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
if (!string.IsNullOrEmpty(specificBlog))
command.Parameters.AddWithValue("@blog", specificBlog);
if (specificBlogs != null && specificBlogs.Count > 0)
{
for (int i = 0; i < specificBlogs.Count; i++)
command.Parameters.AddWithValue($"@blog{i}", specificBlogs[i]);
}
else if (!ignoreCooldown)
command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
+54 -38
View File
@@ -170,9 +170,10 @@ namespace URLNotesGrabberCORE
}
string? containsListSetting = settings.GetValue<string>("ContainsList");
if (string.IsNullOrEmpty(containsListSetting))
List<string> contains = (containsListSetting ?? string.Empty)
.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToList();
if (contains.Count == 0)
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
List<string> contains = containsListSetting.Split(',').ToList();
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
@@ -415,9 +416,15 @@ namespace URLNotesGrabberCORE
break;
case "--likes":
string likeBlog = args.Length > 1 ? args[1] : null;
List<string> likeBlogs = new List<string>();
for (int j = 1; j < args.Length; j++)
{
string arg = args[j];
if (arg.StartsWith("--")) break; // stop at next flag
likeBlogs.Add(arg);
}
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
CollectLikes(likeBlogs, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
break;
case "--urldump":
@@ -509,7 +516,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
Console.WriteLine("--likes [blog1 [blog2 ...]]\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. One or more blog names restricts to those blogs.");
Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
@@ -913,14 +920,14 @@ namespace URLNotesGrabberCORE
}
}
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
static async Task CollectLikes(List<string> specificBlogs, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
{
try
{
DataAccess.EnsureBlogsLikesColumnsExist();
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlogs, cooldownDays, ignoreCooldown);
if (blogsToProcess.Count == 0)
{
@@ -965,7 +972,8 @@ namespace URLNotesGrabberCORE
if (isRefresh) cursor = 0;
string mode = isRefresh ? "REFRESH" : "BACKFILL";
Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
int pct = blogsToProcess.Count > 0 ? (((i + 1) * 100) / blogsToProcess.Count) : 0;
Console.WriteLine($"[{i + 1}/{blogsToProcess.Count} ({pct}%) | {remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
bool hasMoreLikes = true;
bool isFirstPage = true;
@@ -1592,42 +1600,49 @@ if (shouldInsert)
var urls = new List<string>();
var reblog = new ReblogRecord();
// A record without downloaded files is only complete once the next
// "Post id:" line (or end of file) is reached, so it is imported then.
void ImportRecordWithoutDownloadedFiles(ReblogRecord record, ref int added)
{
if (record.reblogName != "." && record.postID != "." && record.date != "." && record.reblogURL != "." && record.downloadedFiles == ".")
{
if (!record.reblogURL.Contains("deactivated")
&& record.reblogURL.Length != 0
&& ContainsAny(record.reblogURL, contains))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
string curDir = NormalizeBlogFolderName(currentDir.Name);
urls.Add(string.Format("{0}\t{1}", curDir, record.postID));
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
DataAccess.AddPost(curDir, long.Parse(record.postID), record.reblogURL, record.date, record.postURL, record.slug, record.reblogKey,
record.reblogName, record.summary, record.quote, record.body, record.tags, record.link, record.photoURL,
record.photoCaption, record.downloadedFiles, record.audioCaption, record.question, record.answer,
record.title, false, rootURL: record.rootURL, postType: filePostType);
recordImportStopwatch.Stop();
added++;
directoryRecordsImported++;
if (logRecordImports)
Console.WriteLine($"[Record Import] {curDir}/{record.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={added}");
// Output hyperlink and post date
//Console.WriteLine($"https://{record.reblogName}.tumblr.com/post/{record.postID} - {record.date}");
}
}
}
string[] fileLines = File.ReadAllLines(file);
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
{
string line = fileLines[lineIndex];
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
{
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
{
if (!reblog.reblogURL.Contains("deactivated")
&& reblog.reblogURL.Length != 0
&& ContainsAny(reblog.reblogURL, contains))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
string curDir = NormalizeBlogFolderName(currentDir.Name);
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, false, rootURL: reblog.rootURL, postType: filePostType);
recordImportStopwatch.Stop();
postsAdded++;
directoryRecordsImported++;
if (logRecordImports)
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
// Output hyperlink and post date
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
}
}
ImportRecordWithoutDownloadedFiles(reblog, ref postsAdded);
reblog = new ReblogRecord();
reblog.postID = line.Substring(9).Trim();
}
@@ -1777,6 +1792,7 @@ if (shouldInsert)
}
}
}
ImportRecordWithoutDownloadedFiles(reblog, ref postsAdded);
urls.Sort();