Add byLikes tracking to blogs and posts in DataAccess
Introduce byLikes parameter to AddBlog, AddPost, and UpdatePost methods, updating SQL logic and schema to store this flag. Ensure all relevant calls and SQL statements handle the new ByLikes column, allowing tracking of whether entries were added "by likes." Also standardize hasImage boolean handling in SQL. Add Tumblr likes fetching and DB tracking support Implemented -likes command to fetch/process Tumblr blog likes. Added DB columns and logic to track likes progress (LikesPulled, LikesCursor). Integrated API call, pagination, and rate-limit handling for likes. Extended AddBlog/AddPost/UpdatePost for likes-related fields. Added DateModified tracking to DB operations. Improved error handling and updated launch/app settings.
This commit is contained in:
@@ -75,6 +75,8 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
|
||||
|
||||
Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param");
|
||||
|
||||
break;
|
||||
|
||||
case "-parse":
|
||||
@@ -185,6 +187,11 @@ namespace URLNotesGrabberCORE
|
||||
CollectMissingReplyText().GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
case "-likes":
|
||||
string likeBlog = args.Length > 1 ? args[1] : null;
|
||||
CollectLikes(likeBlog, contains).GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
default:
|
||||
Console.WriteLine("** Unknown Command ** " + args[0]);
|
||||
break;
|
||||
@@ -442,6 +449,261 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
static async Task CollectLikes(string specificBlog, List<string> contains)
|
||||
{
|
||||
try
|
||||
{
|
||||
DataAccess.EnsureBlogsLikesColumnsExist();
|
||||
|
||||
Console.WriteLine("Starting collection of likes...");
|
||||
var blogsToProcess = DataAccess.GetBlogsForLikes(specificBlog);
|
||||
|
||||
if (blogsToProcess.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No blogs found to process likes for.");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Found {blogsToProcess.Count} blogs to process likes.");
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
foreach (var blogInfo in blogsToProcess)
|
||||
{
|
||||
string blogName = blogInfo.Item1;
|
||||
int likesPulled = blogInfo.Item2;
|
||||
long cursor = blogInfo.Item3;
|
||||
long parsedForBlog = 0;
|
||||
long matchedForBlog = 0;
|
||||
int likedCountForBlog = 0;
|
||||
|
||||
Console.WriteLine($"Processing likes for blog: {blogName} | Cursor: {cursor}");
|
||||
|
||||
bool hasMoreLikes = true;
|
||||
|
||||
while (hasMoreLikes)
|
||||
{
|
||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||
if (!lease.IsAcquired)
|
||||
{
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
return; // Might want to sleep instead, but this matches other functions
|
||||
}
|
||||
|
||||
var response = await APIAccess.GrabLikes(blogName, cursor);
|
||||
|
||||
if (response?.statusCode == "NotFound" || (response?.meta != null && response.meta.status == 404))
|
||||
{
|
||||
Console.WriteLine($"API returned 404 Not Found for {blogName} Likes");
|
||||
// Set pulled = 1 to skip in future
|
||||
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor);
|
||||
break;
|
||||
}
|
||||
|
||||
if (response == null || response.statusCode == "TooManyRequests")
|
||||
{
|
||||
int retry = response?.retryInSeconds ?? 60;
|
||||
if (retry <= 0)
|
||||
retry = 60;
|
||||
|
||||
int remaining = retry;
|
||||
DateTime retryUntil = DateTime.Now.AddSeconds(retry);
|
||||
while (remaining > 0)
|
||||
{
|
||||
Console.WriteLine("Sleeping for {0} more seconds, until {1}", remaining, retryUntil.ToShortTimeString());
|
||||
int sleepSeconds = Math.Min(60, remaining);
|
||||
Thread.Sleep(sleepSeconds * 1000);
|
||||
remaining -= sleepSeconds;
|
||||
}
|
||||
continue; // Retry same cursor
|
||||
}
|
||||
|
||||
if (response?.response?.liked_posts == null || response.response.liked_posts.Count == 0)
|
||||
{
|
||||
Console.WriteLine($"[Likes] No more likes found for {blogName}. Marking complete.");
|
||||
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursor); // Done parsing
|
||||
hasMoreLikes = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if (response.response.liked_count > 0)
|
||||
likedCountForBlog = response.response.liked_count;
|
||||
|
||||
Console.WriteLine($"[Likes] Fetched {response.response.liked_posts.Count} likes for {blogName}");
|
||||
|
||||
foreach (var post in response.response.liked_posts)
|
||||
{
|
||||
parsedForBlog++;
|
||||
|
||||
long postID = 0;
|
||||
try { postID = Convert.ToInt64(post.id); } catch { continue; }
|
||||
|
||||
string authorBlog = post.blog_name?.ToString() ?? ".";
|
||||
string postURL = post.post_url?.ToString() ?? ".";
|
||||
string date = post.date?.ToString() ?? ".";
|
||||
|
||||
// Legacy format fields
|
||||
string body = post.body?.ToString() ?? ".";
|
||||
string summary = post.summary?.ToString() ?? ".";
|
||||
string slug = post.slug?.ToString() ?? ".";
|
||||
string reblogURL = post.source_url?.ToString() ?? ".";
|
||||
string reblogName = post.reblogged_from_name?.ToString() ?? post.source_title?.ToString() ?? ".";
|
||||
string rootBlogName = post.reblogged_root_name?.ToString() ?? ".";
|
||||
string rootURL = post.reblogged_root_url?.ToString() ?? ".";
|
||||
|
||||
// Quick check for tags
|
||||
string tags = ".";
|
||||
if (post.tags != null)
|
||||
{
|
||||
try { tags = string.Join(", ", post.tags); }
|
||||
catch { }
|
||||
}
|
||||
|
||||
bool hasImage = false;
|
||||
string photoURL = ".";
|
||||
string photoCaption = ".";
|
||||
|
||||
if (post.photos != null)
|
||||
{
|
||||
hasImage = true;
|
||||
try
|
||||
{
|
||||
var firstPhoto = post.photos[0];
|
||||
if (firstPhoto != null)
|
||||
{
|
||||
if (firstPhoto.original_size != null)
|
||||
photoURL = firstPhoto.original_size.url?.ToString() ?? ".";
|
||||
photoCaption = firstPhoto.caption?.ToString() ?? ".";
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
else if (body.IndexOf("<img", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
hasImage = true;
|
||||
}
|
||||
|
||||
// Optional fields
|
||||
string quote = ".";
|
||||
string audioCaption = ".";
|
||||
string question = ".";
|
||||
string answer = ".";
|
||||
string title = post.title?.ToString() ?? ".";
|
||||
string downloadedFiles = ".";
|
||||
string reblogKey = post.reblog_key?.ToString() ?? ".";
|
||||
string link = ".";
|
||||
|
||||
// Only insert if any of the data contains strings from ContainsList
|
||||
bool shouldInsert = false;
|
||||
string matchedFieldName = string.Empty;
|
||||
|
||||
// Let's check a few fields that usually have URLs or relevant info that might match ContainsList
|
||||
var fieldsToCheck = new (string Name, string Value)[] {
|
||||
("postURL", postURL),
|
||||
("authorBlog", authorBlog),
|
||||
("reblogURL", reblogURL),
|
||||
("reblogName", reblogName),
|
||||
("rootBlogName", rootBlogName),
|
||||
("rootURL", rootURL),
|
||||
("summary", summary),
|
||||
("body", body),
|
||||
("tags", tags),
|
||||
("photoURL", photoURL),
|
||||
("photoCaption", photoCaption),
|
||||
("quote", quote),
|
||||
("question", question),
|
||||
("answer", answer),
|
||||
("downloadedFiles", downloadedFiles),
|
||||
("slug", slug)
|
||||
};
|
||||
|
||||
foreach (var field in fieldsToCheck)
|
||||
{
|
||||
if (ContainsAny(field.Value, contains))
|
||||
{
|
||||
shouldInsert = true;
|
||||
matchedFieldName = field.Name;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldInsert)
|
||||
{
|
||||
matchedForBlog++;
|
||||
Console.WriteLine($"[Likes] Match | Author: {authorBlog} | PostID: {postID} | Field: {matchedFieldName}");
|
||||
await Task.Delay(3000);
|
||||
DataAccess.AddPost(authorBlog, postID, reblogURL, date, postURL, slug, reblogKey,
|
||||
reblogName, summary, quote, body, tags, link, photoURL,
|
||||
photoCaption, downloadedFiles, audioCaption, question, answer,
|
||||
title, hasImage, true, rootBlogName: rootBlogName, rootURL: rootURL);
|
||||
}
|
||||
}
|
||||
|
||||
if (likedCountForBlog > 0)
|
||||
{
|
||||
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
||||
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
||||
Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}");
|
||||
}
|
||||
|
||||
// Determine the next BeforeCursor.
|
||||
long nextCursor = 0;
|
||||
if (response.response._links?.next?.query_params != null)
|
||||
{
|
||||
long.TryParse(response.response._links.next.query_params.before ?? "0", out nextCursor);
|
||||
}
|
||||
|
||||
// Persist cursor progress after every page so resume is always up-to-date
|
||||
long cursorToPersist = nextCursor > 0 ? nextCursor : cursor;
|
||||
DataAccess.UpdateBlogLikesStatus(blogName, 0, cursorToPersist);
|
||||
|
||||
if (nextCursor > 0)
|
||||
{
|
||||
cursor = nextCursor;
|
||||
Console.WriteLine($"[Likes] Pagination Next Cursor: {cursor}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"[Likes] No further pagination items. Done with {blogName}.");
|
||||
DataAccess.UpdateBlogLikesStatus(blogName, 1, cursorToPersist); // Mark as completely pulled
|
||||
hasMoreLikes = false;
|
||||
}
|
||||
|
||||
await Task.Delay(1000); // 1-second delay between pages
|
||||
}
|
||||
|
||||
if (likedCountForBlog > 0)
|
||||
{
|
||||
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
||||
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
||||
Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Likes collection complete.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error collecting likes: {ex.Message}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
|
||||
{
|
||||
try
|
||||
|
||||
Reference in New Issue
Block a user