feat: add [X remaining] progress counter to likes mode output
This commit is contained in:
+24
-5
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="C:/Users/jim/Nextcloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="4253"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="ApiKeyPoolMeta" custom_title="0" dock_id="4" table="4,14:mainApiKeyPoolMeta"/><dock_state state="000000ff00000000fd00000001000000020000077400000387fc0100000006fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc00000000000007740000000000fffffffaffffffff0100000001fb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011e00ffffff000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings/></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="C:/Users/jim/Nextcloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="4305"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077200000379fc0100000006fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000000007720000011700fffffffb000000160064006f0063006b00420072006f00770073006500340100000000000005f40000000000000000000007720000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="ApiKeyPoolMeta" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="29"/><column index="2" value="64"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/><column index="5" value="51"/><column index="6" value="71"/><column index="7" value="85"/><column index="8" value="156"/><column index="9" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="28" mode="1"/></sort><column_widths><column index="1" value="241"/><column index="2" value="148"/><column index="3" value="126"/><column index="4" value="300"/><column index="5" value="75"/><column index="6" value="187"/><column index="7" value="159"/><column index="8" value="75"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="78"/><column index="12" value="249"/><column index="13" value="300"/><column index="14" value="53"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="41"/><column index="18" value="75"/><column index="19" value="96"/><column index="20" value="300"/><column index="21" value="96"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="42"/><column index="25" value="60"/><column index="26" value="218"/><column index="27" value="920"/><column index="28" value="156"/><column index="29" value="156"/></column_widths><filter_values><column index="24" value="=1"/><column index="28" value=">2026-05-06 20:00:01"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
|
||||
SET HasNotesGathered = 0
|
||||
WHERE (BlogName, PostID) IN (
|
||||
SELECT p.BlogName, p.PostID
|
||||
@@ -31,9 +31,10 @@ blogname in
|
||||
|
||||
)</sql><sql name="New Notes">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
|
||||
from Notes
|
||||
where --type like 'r%' and
|
||||
DatetimeCrawled <> '2026-04-30 09:25:43'
|
||||
order by DatetimeCrawled desc, TimeStamp desc</sql><sql name="Pull Blogs">SELECT distinct
|
||||
where
|
||||
DatetimeCrawled > '2026-05-14 02:50:05' --and type like 'r%'
|
||||
order by DatetimeCrawled desc</sql><sql name="Pull Blogs*">SELECT distinct␍
|
||||
'''' || blogname || ''',',
|
||||
blogs.*
|
||||
, blogname || '.tumblr.com'
|
||||
FROM
|
||||
@@ -45,4 +46,22 @@ WHERE
|
||||
order by
|
||||
Notes.Type desc,
|
||||
DateAdded desc
|
||||
LIMIT 100;</sql><current_tab id="1"/></tab_sql></sqlb_project>
|
||||
LIMIT 100;</sql><sql name="SQL 7">WITH ReplyCounts AS (
|
||||
SELECT
|
||||
NoteBlogName,
|
||||
COUNT(DISTINCT replyText) AS DistinctReplyCount
|
||||
FROM Notes
|
||||
where replyText <> '.'
|
||||
GROUP BY NoteBlogName
|
||||
)
|
||||
SELECT
|
||||
n.RootBlogName || '.tumblr.com/post/' || n.PostID AS PostURL, postid,
|
||||
n.NoteBlogName,
|
||||
n.replyText,
|
||||
c.DistinctReplyCount
|
||||
FROM Notes n
|
||||
JOIN ReplyCounts c ON n.NoteBlogName = c.NoteBlogName
|
||||
where replyText <> '.' and type <> 'reply'
|
||||
--AND N.NoteBlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592',
|
||||
--'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' )
|
||||
order by c.DistinctReplyCount desc, n.NoteBlogName, n.DateModified desc, replyText, RootBlogName, PostID</sql><current_tab id="3"/></tab_sql></sqlb_project>
|
||||
|
||||
@@ -769,8 +769,9 @@ namespace URLNotesGrabberCORE
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
foreach (var blogInfo in blogsToProcess)
|
||||
foreach (int i, var blogInfo in blogsToProcess)
|
||||
{
|
||||
int remaining = blogsToProcess.Count - i - 1;
|
||||
string blogName = blogInfo.Item1;
|
||||
int likesPulled = blogInfo.Item2;
|
||||
long cursor = blogInfo.Item3;
|
||||
@@ -788,7 +789,7 @@ namespace URLNotesGrabberCORE
|
||||
if (isRefresh) cursor = 0;
|
||||
|
||||
string mode = isRefresh ? "REFRESH" : "BACKFILL";
|
||||
Console.WriteLine($"Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
|
||||
Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}");
|
||||
|
||||
bool hasMoreLikes = true;
|
||||
bool isFirstPage = true;
|
||||
@@ -844,7 +845,7 @@ namespace URLNotesGrabberCORE
|
||||
if (response.response.liked_count > 0)
|
||||
likedCountForBlog = response.response.liked_count;
|
||||
|
||||
Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}");
|
||||
Console.WriteLine($"[Likes] [Fetched {response.response.liked_posts.Count}] {blogName}");
|
||||
|
||||
bool crossedHighWaterMark = false;
|
||||
|
||||
@@ -957,11 +958,11 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldInsert)
|
||||
{
|
||||
if (shouldInsert)
|
||||
{
|
||||
matchedForBlog++;
|
||||
if (isRefresh) newInsertedInRefresh++;
|
||||
Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
|
||||
Console.WriteLine($"[Likes] [Matched] {blogName} | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
|
||||
await Task.Delay(3000);
|
||||
DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
|
||||
reblogName, summary, quote, body, tags, link, photoURL,
|
||||
@@ -1023,6 +1024,7 @@ namespace URLNotesGrabberCORE
|
||||
hasMoreLikes = false;
|
||||
}
|
||||
|
||||
Console.WriteLine($"[{remaining} remaining] Done with {blogName}");
|
||||
await Task.Delay(1000); // 1-second delay between pages
|
||||
}
|
||||
|
||||
@@ -1031,7 +1033,7 @@ namespace URLNotesGrabberCORE
|
||||
WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
||||
}
|
||||
|
||||
Console.WriteLine("Likes collection complete.");
|
||||
Console.WriteLine("[Likes] [Likes collection complete]");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user