From 21e848efb7afd515845c06e91a2ef64f75157cf4 Mon Sep 17 00:00:00 2001 From: jim Date: Tue, 19 May 2026 15:48:18 -0500 Subject: [PATCH] feat: add [X remaining] progress counter to likes mode output --- RERUN.sqbpro | 115 +++++++++++++++++++-------------- URLNotesGrabberCORE/Program.cs | 22 ++++--- 2 files changed, 79 insertions(+), 58 deletions(-) diff --git a/RERUN.sqbpro b/RERUN.sqbpro index 882e476..98e48b7 100644 --- a/RERUN.sqbpro +++ b/RERUN.sqbpro @@ -1,48 +1,67 @@ -UPDATE Posts -SET HasNotesGathered = 0 -WHERE (BlogName, PostID) IN ( - SELECT p.BlogName, p.PostID - FROM Posts p - WHERE p.HasNotesGathered = 1 - AND P.notesGatheredDatetime < 1774294520 - AND EXISTS ( - SELECT 1 - FROM Notes n - WHERE n.PostID = p.PostID - AND n.RootBlogName = p.BlogName - --AND n.Type NOT IN ('reblog', 'reply') - ) - ORDER BY P.PostDate ASC - --LIMIT 500 -);select * -from Blogs ---update blogs set HasBeenOutput = 1 -where HasBeenOutput = 0 -AND -blogname in -( -'teaberrybee', -'reddevilgoddesstoo', -'waywardog13', -'wzjustbrowsing-blog', -'lewerta', -'nudenymph', -'caylachief' - -)select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch') -from Notes -where --type like 'r%' and -DatetimeCrawled <> '2026-04-30 09:25:43' -order by DatetimeCrawled desc, TimeStamp descSELECT distinct - blogs.* - , blogname || '.tumblr.com' -FROM - Blogs - inner JOIN - Notes on notes.noteBlogName = blogs.BlogName -WHERE - HasBeenOutput = 0 and type = 'reblog' -order by - Notes.Type desc, - DateAdded desc -LIMIT 100; +
UPDATE Posts +SET HasNotesGathered = 0 +WHERE (BlogName, PostID) IN ( + SELECT p.BlogName, p.PostID + FROM Posts p + WHERE p.HasNotesGathered = 1 + AND P.notesGatheredDatetime < 1774294520 + AND EXISTS ( + SELECT 1 + FROM Notes n + WHERE n.PostID = p.PostID + AND n.RootBlogName = p.BlogName + --AND n.Type NOT IN ('reblog', 'reply') + ) + ORDER BY P.PostDate ASC + --LIMIT 500 +);select * +from Blogs +--update blogs set HasBeenOutput = 1 +where HasBeenOutput = 0 +AND +blogname in +( +'teaberrybee', +'reddevilgoddesstoo', +'waywardog13', +'wzjustbrowsing-blog', +'lewerta', +'nudenymph', +'caylachief' + +)select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch') +from Notes +where +DatetimeCrawled > '2026-05-14 02:50:05' --and type like 'r%' +order by DatetimeCrawled descSELECT distinct + '''' || blogname || ''',', + blogs.* + , blogname || '.tumblr.com' +FROM + Blogs + inner JOIN + Notes on notes.noteBlogName = blogs.BlogName +WHERE + HasBeenOutput = 0 and type = 'reblog' +order by + Notes.Type desc, + DateAdded desc +LIMIT 100;WITH ReplyCounts AS ( + SELECT + NoteBlogName, + COUNT(DISTINCT replyText) AS DistinctReplyCount + FROM Notes + where replyText <> '.' + GROUP BY NoteBlogName +) +SELECT + n.RootBlogName || '.tumblr.com/post/' || n.PostID AS PostURL, postid, + n.NoteBlogName, + n.replyText, + c.DistinctReplyCount +FROM Notes n +JOIN ReplyCounts c ON n.NoteBlogName = c.NoteBlogName +where replyText <> '.' and type <> 'reply' +--AND N.NoteBlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592', +--'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' ) +order by c.DistinctReplyCount desc, n.NoteBlogName, n.DateModified desc, replyText, RootBlogName, PostID diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index af4dc38..43ba896 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -769,8 +769,9 @@ namespace URLNotesGrabberCORE AutoReplenishment = true }); - foreach (var blogInfo in blogsToProcess) + foreach (int i, var blogInfo in blogsToProcess) { + int remaining = blogsToProcess.Count - i - 1; string blogName = blogInfo.Item1; int likesPulled = blogInfo.Item2; long cursor = blogInfo.Item3; @@ -788,7 +789,7 @@ namespace URLNotesGrabberCORE if (isRefresh) cursor = 0; string mode = isRefresh ? "REFRESH" : "BACKFILL"; - Console.WriteLine($"Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}"); + Console.WriteLine($"[{remaining} remaining] Processing likes for blog: {blogName} | Mode: {mode} | Cursor: {cursor} | HighWaterMark: {storedNewestTs}"); bool hasMoreLikes = true; bool isFirstPage = true; @@ -844,7 +845,7 @@ namespace URLNotesGrabberCORE if (response.response.liked_count > 0) likedCountForBlog = response.response.liked_count; - Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}"); + Console.WriteLine($"[Likes] [Fetched {response.response.liked_posts.Count}] {blogName}"); bool crossedHighWaterMark = false; @@ -957,16 +958,16 @@ namespace URLNotesGrabberCORE } } - if (shouldInsert) - { +if (shouldInsert) +{ matchedForBlog++; if (isRefresh) newInsertedInRefresh++; - Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}"); + Console.WriteLine($"[Likes] [Matched] {blogName} | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}"); await Task.Delay(3000); DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey, - reblogName, summary, quote, body, tags, link, photoURL, - photoCaption, downloadedFiles, audioCaption, question, answer, - title, hasImage, true, rootBlogName: rootBlogName, rootURL: rootURL); + reblogName, summary, quote, body, tags, link, photoURL, + photoCaption, downloadedFiles, audioCaption, question, answer, + title, hasImage, true, rootBlogName: rootBlogName, rootURL: rootURL); } } @@ -1023,6 +1024,7 @@ namespace URLNotesGrabberCORE hasMoreLikes = false; } + Console.WriteLine($"[{remaining} remaining] Done with {blogName}"); await Task.Delay(1000); // 1-second delay between pages } @@ -1031,7 +1033,7 @@ namespace URLNotesGrabberCORE WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog); } - Console.WriteLine("Likes collection complete."); + Console.WriteLine("[Likes] [Likes collection complete]"); } catch (Exception ex) {