diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index af5af0d..fdc92a8 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -81,6 +81,57 @@ namespace URLNotesGrabberCORE return dateTime; } + public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + // Check if replyText column exists + string checkSql = "PRAGMA table_info(Notes);"; + using (SQLiteCommand command = new SQLiteCommand(checkSql, connection)) + { + using (SQLiteDataReader reader = command.ExecuteReader()) + { + bool columnExists = false; + while (reader.Read()) + { + string columnName = reader.GetString(1); + if (columnName.Equals("replyText", StringComparison.OrdinalIgnoreCase)) + { + columnExists = true; + break; + } + } + + if (!columnExists) + { + // Add the column if it doesn't exist + connection.Close(); + connection.Open(); + + string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';"; + using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection)) + { + addCommand.ExecuteNonQuery(); + Console.WriteLine("[Migration] Added replyText column to Notes table"); + } + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error checking/creating replyText column: {ex.Message}"); + } + finally + { + connection.Close(); + } + } + #region Adds public static void AddBlog(string blogName, string DBPath = @"TL.db") { @@ -93,10 +144,14 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "INSERT OR IGNORE INTO Blogs (BlogName) values('" + blogName + "')"; - SQLiteCommand command = new SQLiteCommand(sql, connection); - - command.ExecuteNonQuery(); + // Insert BlogName and DateAdded (current UTC datetime) + string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded) VALUES (@BlogName, @DateAdded)"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@BlogName", blogName); + command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.ExecuteNonQuery(); + } // Console.WriteLine("+ " + blogName); } catch (Exception ex) @@ -226,7 +281,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)"; + string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@rootBlogName", rootBlogName); @@ -234,6 +289,7 @@ namespace URLNotesGrabberCORE command.Parameters.AddWithValue("@PostID", postID); command.Parameters.AddWithValue("@TimeStamp", timestamp); command.Parameters.AddWithValue("@Type", type ?? string.Empty); + command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); command.ExecuteNonQuery(); } @@ -272,9 +328,9 @@ namespace URLNotesGrabberCORE connection.Open(); string sql = "SELECT " + - " Posts.BlogName, " + Environment.NewLine + + " MAX(Posts.BlogName) as BlogName, " + Environment.NewLine + " Posts.PostID, " + Environment.NewLine + - " Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine + + " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine + " Posts.NotesGatheredDatetime, " + Environment.NewLine + " CNT.CNT " + Environment.NewLine + "FROM " + Environment.NewLine + @@ -283,7 +339,7 @@ namespace URLNotesGrabberCORE " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + " LEFT OUTER JOIN " + Environment.NewLine + " ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " + - "WHERE NotFound = 0 " + Environment.NewLine; + "WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine; if (withoutNotesOnly) sql += " and HasNotesGathered = 0 " + Environment.NewLine; @@ -298,12 +354,12 @@ namespace URLNotesGrabberCORE sql += "GROUP BY " + Environment.NewLine + " Posts.BlogName, Posts.PostID " + Environment.NewLine + "ORDER BY " + Environment.NewLine + - " notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine; + " notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine; - Console.WriteLine(withoutNotesOnly); - Console.WriteLine(sql); - Console.Write(">"); //Console.ReadKey(); - Thread.Sleep(250); + //Console.WriteLine(withoutNotesOnly); + //Console.WriteLine(sql); + //Console.Write(">"); //Console.ReadKey(); + //Thread.Sleep(250); using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -382,6 +438,107 @@ namespace URLNotesGrabberCORE return posts; } + public static List> GetRepliesWithMissingText(string DBPath = @"TL.db", int limit = 50) + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + List> posts = new List>(); + + try + { + connection.Open(); + + string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID, + MAX(Notes.timestamp) as LatestTimestamp + FROM Notes + WHERE Notes.type = 'reply' + AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.') + GROUP BY Notes.RootBlogName, Notes.PostID + ORDER BY LatestTimestamp ASC + LIMIT @limit"; + + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@limit", limit); + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + Tuple post = default; + string blog = null; + long id = 0; + + blog = reader.GetString(reader.GetOrdinal("blogName")); + id = reader.GetInt64(reader.GetOrdinal("PostID")); + + post = new Tuple(blog, id); + posts.Add(post); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error getting replies with missing text: {ex.Message}"); + } + finally + { + connection.Close(); + } + return posts; + } + + public static List> GetRepliesWithFilledText(string DBPath = @"TL.db", int limit = 1) + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + List> posts = new List>(); + + try + { + connection.Open(); + + string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID, + MAX(Notes.timestamp) as LatestTimestamp + FROM Notes + WHERE Notes.type = 'reply' + AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.') + GROUP BY Notes.RootBlogName, Notes.PostID + ORDER BY LatestTimestamp ASC + LIMIT @limit"; + + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@limit", limit); + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + Tuple post = default; + string blog = null; + long id = 0; + long timestamp = 0; + + blog = reader.GetString(reader.GetOrdinal("blogName")); + id = reader.GetInt64(reader.GetOrdinal("PostID")); + timestamp = reader.GetInt64(reader.GetOrdinal("LatestTimestamp")); + timestamp++; + + post = new Tuple(blog, id, timestamp); + posts.Add(post); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error getting replies with filled text: {ex.Message}"); + } + finally + { + connection.Close(); + } + return posts; + } + public static int GetAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -517,7 +674,8 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID"; + //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID"; + string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE PostID = @PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds()); @@ -760,6 +918,87 @@ namespace URLNotesGrabberCORE return APICount; } + + public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; + string sql = "UPDATE Notes SET replyText = @replyText WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@replyText", replyText ?? "."); + command.Parameters.AddWithValue("@rootBlogName", rootBlogName); + command.Parameters.AddWithValue("@PostID", postID); + command.Parameters.AddWithValue("@noteBlogName", noteBlogName); + command.Parameters.AddWithValue("@TimeStamp", timestamp); + int rowsAffected = command.ExecuteNonQuery(); + + if (rowsAffected == 0) + { + Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}"); + Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}"); + Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}"); + } + else + { + Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}"); + } + } + } + catch (Exception ex) + { + Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}"); + Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}"); + } + finally + { + connection.Close(); + } + } + + public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + + try + { + connection.Open(); + + string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + command.Parameters.AddWithValue("@replyText", replyText ?? "."); + command.Parameters.AddWithValue("@rootBlogName", rootBlogName); + command.Parameters.AddWithValue("@PostID", postID); + int rowsAffected = command.ExecuteNonQuery(); + + if (rowsAffected == 0) + { + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}"); + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}"); + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}"); + } + else + { + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'"); + } + } + } + catch (Exception ex) + { + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}"); + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}"); + } + finally + { + connection.Close(); + } + } #endregion Updates } @@ -784,30 +1023,33 @@ namespace URLNotesGrabberCORE private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ?? throw new InvalidOperationException("ConsumerSecret is not configured"); private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ?? - throw new InvalidOperationException("OAuthToken is not configured"); + throw new InvalidOperationException("OAuthToken is not configured"); private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ?? throw new InvalidOperationException("OAuthTokenSecret is not configured"); public static async Task GrabNotes(string blog, long ID, string? timestamp = null) { - var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]"; + var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all"; URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString()); if (!string.IsNullOrEmpty(timestamp)) { URL += "&before_timestamp=" + timestamp; await Task.Delay(100); } - var client = new RestClient(URL); - var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, - consumerSecret: ConsumerSecret, - token: OAuthToken, - tokenSecret: OAuthTokenSecret, - OAuthSignatureMethod.HmacSha1 - ); + + // Create a new RestClient for each request to ensure fresh OAuth signatures + using (var client = new RestClient(URL)) + { + var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, + consumerSecret: ConsumerSecret, + token: OAuthToken, + tokenSecret: OAuthTokenSecret, + OAuthSignatureMethod.HmacSha1 + ); - client.Authenticator = oAuth1; - var request = new RestRequest(URL, Method.Get); - var response = await client.ExecuteAsync(request); + client.Authenticator = oAuth1; + var request = new RestRequest(URL, Method.Get); + var response = await client.ExecuteAsync(request); var myJsonResponse = response.Content ?? string.Empty; Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}"); @@ -962,11 +1204,75 @@ namespace URLNotesGrabberCORE myDeserializedClass.statusCode = "TooManyRequests"; } } - } + } } } return myDeserializedClass; + } + } + + public static async Task GrabPostWithReplies(string blog, long postID, long timestamp) + { + var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}"; + + // Create a new RestClient for each request to ensure fresh OAuth signatures + using (var client = new RestClient(URL)) + { + var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey, + consumerSecret: ConsumerSecret, + token: OAuthToken, + tokenSecret: OAuthTokenSecret, + OAuthSignatureMethod.HmacSha1 + ); + + client.Authenticator = oAuth1; + var request = new RestRequest(URL, Method.Get); + var response = await client.ExecuteAsync(request); + + var myJsonResponse = response.Content ?? string.Empty; + Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}"); + var myDeserializedClass = new PostsRoot(); + + // Check if response is successful and contains JSON + if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse)) + { + string? statusStr = null; + try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; } + Console.WriteLine($"[Reply API] {statusStr}\t{response?.StatusDescription}"); + if (!string.IsNullOrEmpty(statusStr)) + myDeserializedClass.statusCode = statusStr; + return myDeserializedClass; + } + + // Check if response is HTML (error page) instead of JSON + if (myJsonResponse.TrimStart().StartsWith("<")) + { + Console.WriteLine($"[Reply API] Received HTML response (likely error page): {response?.StatusCode}"); + string? statusStr = null; + try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; } + if (!string.IsNullOrEmpty(statusStr)) + myDeserializedClass.statusCode = statusStr; + return myDeserializedClass; + } + + try + { + var deserializedResult = JsonConvert.DeserializeObject(myJsonResponse); + if (deserializedResult != null) + { + myDeserializedClass = deserializedResult; + myDeserializedClass.rawJson = myJsonResponse; + } + } + catch (Exception ex) + { + Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}"); + Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}"); + } + + return myDeserializedClass; + } } } } diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 39d2a5c..80c2bb5 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -26,9 +26,18 @@ namespace URLNotesGrabberCORE if (File.Exists(logPath)) { string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); - string newPath = $"console_output_{timestamp}.log"; + string archiveDirectory = "logs"; + + // Ensure the archive directory exists + if (!Directory.Exists(archiveDirectory)) + { + Directory.CreateDirectory(archiveDirectory); + } + + string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log"); File.Move(logPath, newPath); } + StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true }; DualLogger dualLogger = new DualLogger(Console.Out, fileWriter); Console.SetOut(dualLogger); @@ -37,7 +46,9 @@ namespace URLNotesGrabberCORE if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB { - TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains); + int postsAdded = 0; + TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, ref postsAdded); + Console.WriteLine($"Total posts added: {postsAdded}"); } else { @@ -62,96 +73,19 @@ namespace URLNotesGrabberCORE Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range "); - Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file "); + Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database"); break; + case "-parse": string blogNameToParse = args[1]; - - TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, blogNameToParse); - + int postsAdded = 0; + TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse); + Console.WriteLine($"Total posts added: {postsAdded}"); break; - + case "-test": - #region Manual test - - string blogName = args[1]; - long postID = long.Parse(args[2]); - - var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult(); - List> notes = new List>(); - - // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound - if (response?.meta != null && response.meta.status == 404) - { - Console.WriteLine($"ERROR: Post not found - {blogName}/{postID} (meta.status=404)"); - DataAccess.UpdatePostMarkNotFound(blogName, postID); - break; - } - - // Check for API errors - if (response.statusCode == "NotFound") - { - Console.WriteLine($"ERROR: Post not found - {blogName}/{postID}"); - break; - } - if (response.statusCode == "TooManyRequests") - { - Console.WriteLine($"ERROR: Rate limited - retry in {response.retryInSeconds} seconds"); - break; - } - if (!string.IsNullOrEmpty(response.statusCode)) - { - Console.WriteLine($"ERROR: API returned status code: {response.statusCode}"); - break; - } - - // Check for null response - if (response.response == null) - { - Console.WriteLine("ERROR: API response is null"); - break; - } - - // Check for null notes - if (response.response.notes == null) - { - Console.WriteLine("ERROR: Notes collection is null (post may have 0 notes)"); - break; - } - - Console.WriteLine($"Processing {response.response.notes.Count} notes..."); - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } - - while (response.response._links != null) - { - response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); - - if (response.response?.notes == null) - { - Console.WriteLine("ERROR: Notes collection became null during pagination"); - break; - } - - Console.WriteLine($"Processing {response.response.notes.Count} more notes..."); - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ; - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } - } - - Console.WriteLine("Test completed successfully"); - #endregion - + Console.WriteLine("Test command not implemented"); break; case "-post": @@ -168,7 +102,7 @@ namespace URLNotesGrabberCORE case "-collect": //collect notes from all posts bool withoutNotesOnly = true; - DateTime? beforeDate = null; + DateTime? beforeDate = DateTime.Now; if (args.Length < 2) { @@ -247,10 +181,8 @@ namespace URLNotesGrabberCORE WriteBlogsToFileAll(settings.GetValue("PathOutputBlogs"), false, from, to, top); break; - case "-replies": //colection posts with replies - try { System.IO.File.Delete(settings.GetValue("PathOutputReplies")); } catch { } - WriteRepliesToFile(settings.GetValue("PathOutputBlogs"), false); - + case "-replies": //update reply text + CollectMissingReplyText().GetAwaiter().GetResult(); break; default: @@ -351,16 +283,181 @@ namespace URLNotesGrabberCORE return false; } + static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp) + { + try + { + Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}"); + // Add 2-second delay before API call to avoid server-side rate limiting + await Task.Delay(2000); + var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp); + + if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0) + { + Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}"); + Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}"); + Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}"); + if (postsResponse?.response != null) + { + Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}"); + } + if (!string.IsNullOrEmpty(postsResponse?.rawJson)) + { + Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}"); + } + + // Mark all replies for this post with '?' to indicate API returned no posts + Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response"); + DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?"); + + return; + } + + // Get the first (and should be only) post + var post = postsResponse.response.posts.FirstOrDefault(); + if (post?.notes == null) + { + Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}"); + return; + } + + Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}"); + + // Update each reply with its text + int replyCount = 0; + int skippedCount = 0; + foreach (var note in post.notes) + { + if (note.type == "reply") + { + if (!string.IsNullOrEmpty(note.reply_text)) + { + DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text); + replyCount++; + + // Output the reply text being stored + string displayText = note.reply_text.Length > 100 + ? note.reply_text.Substring(0, 100) + "..." + : note.reply_text; + Console.WriteLine($" [{note.blog_name}] {displayText}"); + } + else + { + skippedCount++; + Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text"); + } + } + } + + if (replyCount > 0) + { + Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}"); + } + else + { + Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})"); + } + + // Mark any remaining replies with '.' as '?' to indicate they were processed but had no text + Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}"); + //int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?"); + //if (cleanupCount > 0) + //{ + // Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}"); + //} + } + catch (Exception ex) + { + Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}"); + Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}"); + } + } + + static async Task CollectMissingReplyText() + { + try + { + // Ensure the replyText column exists + DataAccess.EnsureReplyTextColumnExists(); + + Console.WriteLine("Starting collection of missing reply text..."); + Console.WriteLine("Processing 10 posts at a time."); + + int batchSize = 1; // Process 10 posts per batch + int totalProcessedCount = 0; + + RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions + { + PermitLimit = 300, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + QueueLimit = 1, + Window = TimeSpan.FromMinutes(1), + SegmentsPerWindow = 60, + AutoReplenishment = true + }); + + while (true && totalProcessedCount < batchSize) + { + var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize); + + if (postsWithFilledReplies.Count == 0) + { + Console.WriteLine("No more posts with filled reply text found. Collection complete."); + break; + } + + Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch..."); + int batchProcessedCount = 0; + + foreach (var post in postsWithFilledReplies) + { + using RateLimitLease lease = limiter.AttemptAcquire(1); + if (lease.IsAcquired) + { + Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}"); + await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3); + batchProcessedCount++; + totalProcessedCount++; + + // Add 1 second delay between attempts + await Task.Delay(1000); + } + else + { + Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); + Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total"); + return; + } + } + + Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch."); + } + + Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts."); + } + catch (Exception ex) + { + Console.WriteLine($"Error collecting missing reply text: {ex.Message}"); + Console.WriteLine(ex.ToString()); + } + } + static async Task GrabNotes(Tuple post) { try { int APICount = DataAccess.GetAPICount(); + Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}"); + Thread.Sleep(3000); - Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount); - var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult(); + var allNotes = new List(); + int page = 1; + string beforeTimestamp = post.Item3.ToString(); + bool hasReplies = false; + const int maxPages = 500; + var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult(); - // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return + // Handle 404 and error codes if (response?.meta != null && response.meta.status == 404) { Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)"); @@ -368,8 +465,11 @@ namespace URLNotesGrabberCORE Thread.Sleep(1000); return "NotFound"; } - List> notes = new List>(); - + if (response == null) + { + Console.WriteLine("##### Response is null - API Failure? ###"); + return "FAILURE"; + } if (response.statusCode == "NotFound") { Thread.Sleep(1000); @@ -386,50 +486,56 @@ namespace URLNotesGrabberCORE return response.statusCode; } - if (response.response != null && response.response.notes != null) + // Pagination loop + while (true) { - Console.WriteLine("Notes\t" + response.response.notes.Count); - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } - } - else if (response.response == null) - { - Console.WriteLine("##### Response Null - API Failure? ###"); - Console.WriteLine($"Raw JSON: {response.rawJson}"); - return "FAILURE"; - } - else if (response.response.notes == null) - { - Console.WriteLine("##### Notes Null - WHY? ###"); - return "FAILURE"; - } - else - { - while (response.response != null - && response.response._links != null - && long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3) - { - response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); - if (response.response.notes is null) - { - Console.WriteLine("response.response.notes == null"); - return "NULL NOTES"; - } + int noteCount = response?.response?.notes?.Count ?? 0; + Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}"); - Console.WriteLine("Notes\t" + response.response.notes.Count); - foreach (var note in response.response.notes) - { - note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString(); - //notes.Add(new Tuple(note.blog_url, note.type)); - if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type)) - break; - } + if (response?.response == null) + { + Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}"); + return "FAILURE"; } + if (response.response.notes == null) + { + Console.WriteLine("##### Notes Null - WHY? ###"); + return "FAILURE"; + } + + // Accumulate notes + allNotes.AddRange(response.response.notes); + + // Pagination: check for next + var nextLink = response.response._links?.next; + if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp)) + { + Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s)."); + break; + } + + beforeTimestamp = nextLink.query_params.before_timestamp; + page++; + if (page > maxPages) + { + Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination."); + break; + } + + // Fetch next page + response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult(); + } + + Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}"); + + // Process all accumulated notes + foreach (var note in allNotes) + { + note.reblog_parent_blog_name = post.Item1; + note.post_id = post.Item2.ToString(); + if (note.type == "reply") + hasReplies = true; + DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type); } return "Success"; @@ -438,8 +544,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine(ex.ToString()); } - - return "UNKNOWN"; } @@ -461,15 +565,15 @@ namespace URLNotesGrabberCORE { using (StreamWriter sw = new StreamWriter(outPath, true)) { - foreach (var post in posts) + while (posts.Count > 0) { + var post = posts[0]; // Process the first post in the list string status; using RateLimitLease lease = limiter.AttemptAcquire(1); if (lease.IsAcquired) { Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString()); - // Thread.Sleep(1000); status = await GrabNotes(post); } else @@ -477,10 +581,18 @@ namespace URLNotesGrabberCORE Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); return; } + if (status == "Success") + { DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); + } else + { Console.WriteLine("GrabNotes Result: " + status); + } + + // Re-fetch the updated list after processing the current post + posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate); } } } @@ -491,27 +603,18 @@ namespace URLNotesGrabberCORE } - static void TraverseDirectory(string path, string outPath, List contains, string blogName = "") + static void TraverseDirectory(string path, string outPath, List contains, ref int postsAdded, string blogName = "") { - //if(!ContainsAny(path, contains)) - //{ - // return; - //} // Get all directories in the current directory and sort them alphabetically var directories = Directory.GetDirectories(path); Array.Sort(directories, StringComparer.InvariantCulture); - //using (StreamWriter sw = new StreamWriter(outPath, true)) - //{ - //sw.WriteLine("=====" + path); - //} - foreach (var directory in directories) { - Console.WriteLine("Directory: " + directory); - TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories + TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories } + try { bool headerWasWritten = false; @@ -520,13 +623,6 @@ namespace URLNotesGrabberCORE { if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)) { - //Console.WriteLine($"=====File: {file}"); - - //using (StreamWriter sw = new StreamWriter(outPath, true)) - //{ - //sw.WriteLine("---" + file); - //} - try { var urls = new List(); @@ -534,12 +630,11 @@ namespace URLNotesGrabberCORE foreach (string line in File.ReadLines(file)) { - //if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated")) - if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase)) - { // if there were no files, let's still collect post info + if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) + { if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") { - if (!reblog.reblogURL.Contains(@"deactivated") + if (!reblog.reblogURL.Contains("deactivated") && reblog.reblogURL.Length != 0 && ContainsAny(reblog.reblogURL, contains)) { @@ -548,28 +643,18 @@ namespace URLNotesGrabberCORE { headerWasWritten = true; } - - //if (reblog.reblogURL.Contains("/blog/private") - // || reblog.body.Contains("/blog/private")) - //{ - // Console.WriteLine(reblog.postID); - // Console.WriteLine(reblog.postURL); - // Console.WriteLine(reblog.date); - // Console.WriteLine(reblog.body); - // Console.WriteLine(reblog.reblogKey); - // Console.WriteLine(reblog.reblogURL); - // Console.WriteLine(reblog.reblogName); - // Console.WriteLine(reblog.downloadedFiles); - //} string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, reblog.title, false); + postsAdded++; + + // Output hyperlink and post date + Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}"); } } - reblog = new ReblogRecord(); reblog.postID = line.Substring(9).Trim(); } @@ -659,8 +744,7 @@ namespace URLNotesGrabberCORE if ((ContainsAny(reblog.downloadedFiles, contains) || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0) || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) - // && !ContainsAny(reblog.reblogURL, contains) - && !reblog.reblogURL.Contains(@"deactivated") + //&& !reblog.reblogURL.Contains("deactivated") || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase)) { DirectoryInfo currentDir = new DirectoryInfo(path); @@ -686,6 +770,10 @@ namespace URLNotesGrabberCORE reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, reblog.title, true); + postsAdded++; + + // Output hyperlink and post date + Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}"); } } } diff --git a/URLNotesGrabberCORE/Properties/launchSettings.json b/URLNotesGrabberCORE/Properties/launchSettings.json index 5d6495c..33c48f3 100644 --- a/URLNotesGrabberCORE/Properties/launchSettings.json +++ b/URLNotesGrabberCORE/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "URLNotesGrabberCORE": { "commandName": "Project", - "commandLineArgs": "-test zomb-eh 802499389473079296" + "commandLineArgs": "-collect 1" } } } \ No newline at end of file diff --git a/URLNotesGrabberCORE/ResponseNotes.cs b/URLNotesGrabberCORE/ResponseNotes.cs index 67296fb..fcc1e73 100644 --- a/URLNotesGrabberCORE/ResponseNotes.cs +++ b/URLNotesGrabberCORE/ResponseNotes.cs @@ -83,4 +83,37 @@ namespace URLNotesGrabberCORE public string rawJson { get; set; } } + + // Classes for Posts API endpoint (for reply_text) + public class PostsResponse + { + public List posts { get; set; } + } + + public class Post + { + public long id { get; set; } + public List notes { get; set; } + } + + public class NoteDetail + { + public long timestamp { get; set; } + public string type { get; set; } + public string blog_name { get; set; } + public string reply_text { get; set; } + } + + public class PostsRoot + { + public Meta meta { get; set; } + [JsonConverter(typeof(EmptyArrayOrObjectConverter))] + public PostsResponse response { get; set; } + + public string statusCode { get; set; } + + public int retryInSeconds { get; set; } + + public string rawJson { get; set; } + } } diff --git a/URLNotesGrabberCORE/TL.db b/URLNotesGrabberCORE/TL.db deleted file mode 100644 index 94763ee..0000000 Binary files a/URLNotesGrabberCORE/TL.db and /dev/null differ