Compare commits
14
Commits
49e581c136
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c09c180f1a | ||
|
|
a3a54a12fe | ||
|
|
ed84d6fec0 | ||
|
|
a6522779eb | ||
|
|
c5bdd93708 | ||
|
|
d16ab14691 | ||
|
|
3448d75030 | ||
|
|
445b52f0c8 | ||
|
|
c5676eb084 | ||
|
|
7557c9a168 | ||
|
|
ca99586176 | ||
|
|
f40d824a8c | ||
|
|
bde6506e01 | ||
|
|
06a3f8c142 |
No files matched your search
@@ -1055,6 +1055,7 @@ namespace URLNotesGrabberCORE
|
|||||||
if (toDate.HasValue)
|
if (toDate.HasValue)
|
||||||
command.Parameters.AddWithValue("@toDate", toDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
|
command.Parameters.AddWithValue("@toDate", toDate.Value.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
|
|
||||||
|
int rowCount = 0;
|
||||||
using (SQLiteDataReader reader = command.ExecuteReader())
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
||||||
{
|
{
|
||||||
while (reader.Read())
|
while (reader.Read())
|
||||||
@@ -1114,6 +1115,8 @@ namespace URLNotesGrabberCORE
|
|||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rowCount++;
|
||||||
|
|
||||||
// Exclude if postID is in the exclusion list
|
// Exclude if postID is in the exclusion list
|
||||||
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
|
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
|
||||||
continue;
|
continue;
|
||||||
@@ -1122,6 +1125,11 @@ namespace URLNotesGrabberCORE
|
|||||||
posts.Add(post);
|
posts.Add(post);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int excluded = rowCount - posts.Count;
|
||||||
|
Console.WriteLine(excluded > 0
|
||||||
|
? $" COUNT: {rowCount} ({excluded} excluded, {posts.Count} to collect)"
|
||||||
|
: $" COUNT: {rowCount}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
@@ -1333,7 +1341,7 @@ namespace URLNotesGrabberCORE
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
|
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(List<string> specificBlogs = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
|
||||||
{
|
{
|
||||||
DBPath ??= GetDefaultDbPath();
|
DBPath ??= GetDefaultDbPath();
|
||||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
@@ -1344,61 +1352,48 @@ namespace URLNotesGrabberCORE
|
|||||||
connection.Open();
|
connection.Open();
|
||||||
string sql;
|
string sql;
|
||||||
|
|
||||||
// The two Notes branches below join on Blogs.BlogId, which is NULL for the 168k
|
if (specificBlogs != null && specificBlogs.Count > 0)
|
||||||
// registry rows that have never appeared in a note. The inner join drops them,
|
|
||||||
// which is correct here -- both branches already require a note to exist -- but
|
|
||||||
// it is the wrong shape for anything that lists the registry.
|
|
||||||
if (!string.IsNullOrEmpty(specificBlog))
|
|
||||||
{
|
{
|
||||||
// Specific blog: always process, bypass cooldown
|
// Specific blogs: always process, bypass cooldown
|
||||||
sql = @"SELECT BlogName,
|
var placeholders = string.Join(", ", specificBlogs.Select((_, i) => $"@blog{i}"));
|
||||||
|
sql = $@"SELECT BlogName,
|
||||||
COALESCE(LikesPulled, 0),
|
COALESCE(LikesPulled, 0),
|
||||||
COALESCE(LikesCursor, 0),
|
COALESCE(LikesCursor, 0),
|
||||||
COALESCE(LikesNewestTimestamp, 0)
|
COALESCE(LikesNewestTimestamp, 0)
|
||||||
FROM Blogs
|
FROM Blogs
|
||||||
WHERE BlogName = @blog
|
WHERE BlogName IN ({placeholders})
|
||||||
AND IsActive = 1";
|
AND IsActive = 1";
|
||||||
}
|
}
|
||||||
else if (ignoreCooldown)
|
|
||||||
{
|
|
||||||
// All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
|
|
||||||
sql = @"SELECT B.BlogName,
|
|
||||||
COALESCE(B.LikesPulled, 0),
|
|
||||||
COALESCE(B.LikesCursor, 0),
|
|
||||||
COALESCE(B.LikesNewestTimestamp, 0)
|
|
||||||
FROM Blogs B
|
|
||||||
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
|
|
||||||
WHERE N.TimeStamp >= 1535778000
|
|
||||||
AND N.RootBlogId = B.BlogId
|
|
||||||
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
|
|
||||||
GROUP BY B.BlogName
|
|
||||||
ORDER BY MIN(N.TimeStamp);";
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Backfill-pending OR refresh-due (past cooldown window)
|
// Only blogs that have left a note; least recently refreshed first, then earliest first note
|
||||||
|
string dueFilter = ignoreCooldown ? string.Empty : @"
|
||||||
|
AND (
|
||||||
|
COALESCE(B.LikesPulled, 0) = 0
|
||||||
|
OR COALESCE(B.LikesLastRefreshed, 0)
|
||||||
|
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
|
||||||
|
)";
|
||||||
sql = @"SELECT B.BlogName,
|
sql = @"SELECT B.BlogName,
|
||||||
COALESCE(B.LikesPulled, 0),
|
COALESCE(B.LikesPulled, 0),
|
||||||
COALESCE(B.LikesCursor, 0),
|
COALESCE(B.LikesCursor, 0),
|
||||||
COALESCE(B.LikesNewestTimestamp, 0)
|
COALESCE(B.LikesNewestTimestamp, 0)
|
||||||
FROM Blogs B
|
FROM Blogs B
|
||||||
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
|
INNER JOIN (SELECT N.NoteBlogId, MIN(N.TimeStamp) AS FirstNoteTs
|
||||||
WHERE N.TimeStamp >= 1535778000
|
FROM Notes N" + WhereIsActive("Notes", "N", DBPath) + @"
|
||||||
AND N.RootBlogId = B.BlogId
|
GROUP BY N.NoteBlogId) F ON F.NoteBlogId = B.BlogId
|
||||||
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
|
WHERE B.IsActive = 1" + dueFilter + @"
|
||||||
AND (
|
ORDER BY COALESCE(B.LikesLastRefreshed, 0),
|
||||||
B.LikesPulled = 0
|
F.FirstNoteTs,
|
||||||
OR COALESCE(B.LikesLastRefreshed, 0)
|
B.BlogName;";
|
||||||
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
|
|
||||||
)
|
|
||||||
GROUP BY B.BlogName
|
|
||||||
ORDER BY MIN(N.TimeStamp);";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrEmpty(specificBlog))
|
if (specificBlogs != null && specificBlogs.Count > 0)
|
||||||
command.Parameters.AddWithValue("@blog", specificBlog);
|
{
|
||||||
|
for (int i = 0; i < specificBlogs.Count; i++)
|
||||||
|
command.Parameters.AddWithValue($"@blog{i}", specificBlogs[i]);
|
||||||
|
}
|
||||||
else if (!ignoreCooldown)
|
else if (!ignoreCooldown)
|
||||||
command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
|
command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
|
||||||
|
|
||||||
|
|||||||
@@ -170,9 +170,10 @@ namespace URLNotesGrabberCORE
|
|||||||
}
|
}
|
||||||
|
|
||||||
string? containsListSetting = settings.GetValue<string>("ContainsList");
|
string? containsListSetting = settings.GetValue<string>("ContainsList");
|
||||||
if (string.IsNullOrEmpty(containsListSetting))
|
List<string> contains = (containsListSetting ?? string.Empty)
|
||||||
|
.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToList();
|
||||||
|
if (contains.Count == 0)
|
||||||
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
|
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
|
||||||
List<string> contains = containsListSetting.Split(',').ToList();
|
|
||||||
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
|
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
|
||||||
|
|
||||||
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
||||||
@@ -415,9 +416,15 @@ namespace URLNotesGrabberCORE
|
|||||||
break;
|
break;
|
||||||
|
|
||||||
case "--likes":
|
case "--likes":
|
||||||
string likeBlog = args.Length > 1 ? args[1] : null;
|
List<string> likeBlogs = new List<string>();
|
||||||
|
for (int j = 1; j < args.Length; j++)
|
||||||
|
{
|
||||||
|
string arg = args[j];
|
||||||
|
if (arg.StartsWith("--")) break; // stop at next flag
|
||||||
|
likeBlogs.Add(arg);
|
||||||
|
}
|
||||||
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
|
int cooldownDays = settings.GetValue("LikesRefreshCooldownDays", 7);
|
||||||
CollectLikes(likeBlog, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
|
CollectLikes(likeBlogs, contains, cooldownDays, forceIgnoreCooldown).GetAwaiter().GetResult();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "--urldump":
|
case "--urldump":
|
||||||
@@ -509,7 +516,7 @@ namespace URLNotesGrabberCORE
|
|||||||
|
|
||||||
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
|
Console.WriteLine("--replies\t Fetch and update missing reply text for all replies in database");
|
||||||
|
|
||||||
Console.WriteLine("--likes\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. Optional blog name forces single-blog run.");
|
Console.WriteLine("--likes [blog1 [blog2 ...]]\t Fetch likes: initial backfill for new blogs, incremental refresh for blogs past cooldown. One or more blog names restricts to those blogs.");
|
||||||
|
|
||||||
Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
|
Console.WriteLine("--force\t Ignore refresh cooldowns: with --likes, refresh every fully-backfilled blog; with --collect 1, re-queue already-collected posts without waiting out their cooldown");
|
||||||
|
|
||||||
@@ -913,14 +920,14 @@ namespace URLNotesGrabberCORE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static async Task CollectLikes(string specificBlog, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
|
static async Task CollectLikes(List<string> specificBlogs, List<string> contains, int cooldownDays = 7, bool ignoreCooldown = false)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
DataAccess.EnsureBlogsLikesColumnsExist();
|
DataAccess.EnsureBlogsLikesColumnsExist();
|
||||||
|
|
||||||
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
|
Console.WriteLine($"Starting collection of likes... (cooldown {cooldownDays}d, ignoreCooldown={ignoreCooldown})");
|
||||||
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog, cooldownDays, ignoreCooldown);
|
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlogs, cooldownDays, ignoreCooldown);
|
||||||
|
|
||||||
if (blogsToProcess.Count == 0)
|
if (blogsToProcess.Count == 0)
|
||||||
{
|
{
|
||||||
@@ -965,7 +972,8 @@ namespace URLNotesGrabberCORE
|
|||||||
if (isRefresh) cursor = 0;
|
if (isRefresh) cursor = 0;
|
||||||
|
|
||||||
string mode = isRefresh ? "REFRESH" : "BACKFILL";
|
string mode = isRefresh ? "REFRESH" : "BACKFILL";
|
||||||
Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
|
int pct = blogsToProcess.Count > 0 ? (((i + 1) * 100) / blogsToProcess.Count) : 0;
|
||||||
|
Console.WriteLine($"[{i + 1}/{blogsToProcess.Count} ({pct}%) | {remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
|
||||||
|
|
||||||
bool hasMoreLikes = true;
|
bool hasMoreLikes = true;
|
||||||
bool isFirstPage = true;
|
bool isFirstPage = true;
|
||||||
@@ -1592,42 +1600,49 @@ if (shouldInsert)
|
|||||||
var urls = new List<string>();
|
var urls = new List<string>();
|
||||||
var reblog = new ReblogRecord();
|
var reblog = new ReblogRecord();
|
||||||
|
|
||||||
|
// A record without downloaded files is only complete once the next
|
||||||
|
// "Post id:" line (or end of file) is reached, so it is imported then.
|
||||||
|
void ImportRecordWithoutDownloadedFiles(ReblogRecord record, ref int added)
|
||||||
|
{
|
||||||
|
if (record.reblogName != "." && record.postID != "." && record.date != "." && record.reblogURL != "." && record.downloadedFiles == ".")
|
||||||
|
{
|
||||||
|
if (!record.reblogURL.Contains("deactivated")
|
||||||
|
&& record.reblogURL.Length != 0
|
||||||
|
&& ContainsAny(record.reblogURL, contains))
|
||||||
|
{
|
||||||
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||||
|
if (!headerWasWritten)
|
||||||
|
{
|
||||||
|
headerWasWritten = true;
|
||||||
|
}
|
||||||
|
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
||||||
|
urls.Add(string.Format("{0}\t{1}", curDir, record.postID));
|
||||||
|
|
||||||
|
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
DataAccess.AddPost(curDir, long.Parse(record.postID), record.reblogURL, record.date, record.postURL, record.slug, record.reblogKey,
|
||||||
|
record.reblogName, record.summary, record.quote, record.body, record.tags, record.link, record.photoURL,
|
||||||
|
record.photoCaption, record.downloadedFiles, record.audioCaption, record.question, record.answer,
|
||||||
|
record.title, false, rootURL: record.rootURL, postType: filePostType);
|
||||||
|
recordImportStopwatch.Stop();
|
||||||
|
|
||||||
|
added++;
|
||||||
|
directoryRecordsImported++;
|
||||||
|
if (logRecordImports)
|
||||||
|
Console.WriteLine($"[Record Import] {curDir}/{record.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={added}");
|
||||||
|
|
||||||
|
// Output hyperlink and post date
|
||||||
|
//Console.WriteLine($"https://{record.reblogName}.tumblr.com/post/{record.postID} - {record.date}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
string[] fileLines = File.ReadAllLines(file);
|
string[] fileLines = File.ReadAllLines(file);
|
||||||
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
|
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
|
||||||
{
|
{
|
||||||
string line = fileLines[lineIndex];
|
string line = fileLines[lineIndex];
|
||||||
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
ImportRecordWithoutDownloadedFiles(reblog, ref postsAdded);
|
||||||
{
|
|
||||||
if (!reblog.reblogURL.Contains("deactivated")
|
|
||||||
&& reblog.reblogURL.Length != 0
|
|
||||||
&& ContainsAny(reblog.reblogURL, contains))
|
|
||||||
{
|
|
||||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
||||||
if (!headerWasWritten)
|
|
||||||
{
|
|
||||||
headerWasWritten = true;
|
|
||||||
}
|
|
||||||
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
|
||||||
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
||||||
|
|
||||||
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
||||||
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
||||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
||||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
||||||
reblog.title, false, rootURL: reblog.rootURL, postType: filePostType);
|
|
||||||
recordImportStopwatch.Stop();
|
|
||||||
|
|
||||||
postsAdded++;
|
|
||||||
directoryRecordsImported++;
|
|
||||||
if (logRecordImports)
|
|
||||||
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
|
||||||
|
|
||||||
// Output hyperlink and post date
|
|
||||||
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reblog = new ReblogRecord();
|
reblog = new ReblogRecord();
|
||||||
reblog.postID = line.Substring(9).Trim();
|
reblog.postID = line.Substring(9).Trim();
|
||||||
}
|
}
|
||||||
@@ -1777,6 +1792,7 @@ if (shouldInsert)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ImportRecordWithoutDownloadedFiles(reblog, ref postsAdded);
|
||||||
urls.Sort();
|
urls.Sort();
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user