diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index b288d05..3888c82 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -141,7 +141,7 @@ namespace URLNotesGrabberCORE if (_savedImportPragmas != null) return; - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -177,10 +177,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}"); } - finally - { - connection.Close(); - } } } @@ -192,7 +188,7 @@ namespace URLNotesGrabberCORE if (_savedImportPragmas == null) return; - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -219,7 +215,6 @@ namespace URLNotesGrabberCORE finally { _savedImportPragmas = null; - connection.Close(); } } } @@ -254,7 +249,7 @@ namespace URLNotesGrabberCORE public static void EnsureReplyTextColumnExists(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -298,16 +293,12 @@ namespace URLNotesGrabberCORE // Breakpoint here Console.WriteLine($"Error checking/creating replyText column: {ex.Message}"); } - finally - { - connection.Close(); - } } public static void EnsureBlogsLikesColumnsExist(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -393,10 +384,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}"); } - finally - { - connection.Close(); - } } #region Adds @@ -520,8 +507,12 @@ namespace URLNotesGrabberCORE { if (ownsConnection) connection.Open(); - string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; - SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); + string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID"; + using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); + updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0); + updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + updateCommand.Parameters.AddWithValue("@blogName", blogName); + updateCommand.Parameters.AddWithValue("@postID", postID); updateCommand.ExecuteNonQuery(); } @@ -567,7 +558,7 @@ namespace URLNotesGrabberCORE public static void AddAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -588,10 +579,6 @@ namespace URLNotesGrabberCORE // Breakpoint here //Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null) @@ -600,7 +587,7 @@ namespace URLNotesGrabberCORE //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { } - SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); try { @@ -659,10 +646,6 @@ namespace URLNotesGrabberCORE Console.WriteLine("^^^^^ - SHORTCUT"); } } - finally - { - connection2.Close(); - } return false; } #endregion Adds @@ -678,7 +661,7 @@ namespace URLNotesGrabberCORE public static List> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -856,17 +839,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return posts; } public static List> GetReplies(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -900,17 +879,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return posts; } public static List> GetRepliesWithMissingText(string? DBPath = null, int limit = 50) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -951,17 +926,13 @@ namespace URLNotesGrabberCORE // Breakpoint here Console.WriteLine($"Error getting replies with missing text: {ex.Message}"); } - finally - { - connection.Close(); - } return posts; } public static List> GetRepliesWithFilledText(string? DBPath = null, int? limit = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try @@ -1013,17 +984,13 @@ namespace URLNotesGrabberCORE // Breakpoint here Console.WriteLine($"Error getting replies with filled text: {ex.Message}"); } - finally - { - connection.Close(); - } return posts; } public static int GetAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int count = 0; try { AddAPICount(); } catch { } @@ -1052,17 +1019,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return count; } public static List> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> blogs = new List>(); try @@ -1138,17 +1101,13 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error fetching blogs for likes: {ex.Message}"); } - finally - { - connection.Close(); - } return blogs; } public static List GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); try @@ -1183,17 +1142,13 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return blogs; } public static List GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); try @@ -1228,51 +1183,39 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return blogs; } public static IEnumerable> GetAllPostTextColumns(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); - try + string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { - string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; - using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + using (SQLiteDataReader reader = command.ExecuteReader()) { - using (SQLiteDataReader reader = command.ExecuteReader()) + while (reader.Read()) { - while (reader.Read()) + List rowTexts = new List(); + for (int i = 0; i < reader.FieldCount; i++) { - List rowTexts = new List(); - for (int i = 0; i < reader.FieldCount; i++) + if (!reader.IsDBNull(i)) { - if (!reader.IsDBNull(i)) + var val = reader.GetValue(i); + if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".") { - var val = reader.GetValue(i); - if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".") - { - rowTexts.Add(str); - } + rowTexts.Add(str); } } - if (rowTexts.Count > 0) - { - yield return rowTexts; - } + } + if (rowTexts.Count > 0) + { + yield return rowTexts; } } } } - finally - { - connection.Close(); - connection.Dispose(); - } } #endregion Gets @@ -1282,7 +1225,7 @@ namespace URLNotesGrabberCORE public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1305,16 +1248,12 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1338,10 +1277,6 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } // ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run ----- @@ -1453,7 +1388,7 @@ namespace URLNotesGrabberCORE Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1480,10 +1415,6 @@ namespace URLNotesGrabberCORE return true; } } - finally - { - connection.Close(); - } return false; } @@ -1599,7 +1530,7 @@ namespace URLNotesGrabberCORE public static void UpdateBlogOutput(string blogName, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1619,16 +1550,12 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } } public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -1646,10 +1573,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error updating blog likes status: {ex.Message}"); } - finally - { - connection.Close(); - } } // Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we @@ -1659,7 +1582,7 @@ namespace URLNotesGrabberCORE { if (newestTimestamp <= 0) return; DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -1679,10 +1602,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}"); } - finally - { - connection.Close(); - } } // Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the @@ -1690,7 +1609,7 @@ namespace URLNotesGrabberCORE public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); @@ -1713,16 +1632,12 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}"); } - finally - { - connection.Close(); - } } public static int UpdateAPICount(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int APICount = DataAccess.GetAPICount(); APICount++; @@ -1745,10 +1660,6 @@ namespace URLNotesGrabberCORE if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } - finally - { - connection.Close(); - } return APICount; } @@ -1756,7 +1667,7 @@ namespace URLNotesGrabberCORE public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int rowsAffected = 0; // A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel. @@ -1799,17 +1710,13 @@ namespace URLNotesGrabberCORE Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}"); Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}"); } - finally - { - connection.Close(); - } return rowsAffected; } public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1845,10 +1752,6 @@ namespace URLNotesGrabberCORE Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}"); return 0; } - finally - { - connection.Close(); - } } #endregion Updates @@ -1859,7 +1762,7 @@ namespace URLNotesGrabberCORE public static void EnsureTTFileHelperColumnsExist(string? DBPath = null) { DBPath ??= GetDefaultDbPath(); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1905,10 +1808,6 @@ namespace URLNotesGrabberCORE { Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}"); } - finally - { - connection.Close(); - } } // INSERT-or-UPDATE for a post arriving from a Tumblr text-file export. @@ -2729,7 +2628,7 @@ namespace URLNotesGrabberCORE { minRetrySeconds = 0; if (!_usePool || _overrideKey != null) return false; - if (_keys.Count <= 1) return true; + if (_keys.Count == 0) return false; var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); bool all = true; diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 99343b0..c0bf0ba 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration; using System.Configuration; using System.Threading; using Microsoft.Extensions.Diagnostics.Latency; -using static System.Runtime.InteropServices.JavaScript.JSType; using System.Text.RegularExpressions; namespace URLNotesGrabberCORE @@ -140,7 +139,10 @@ namespace URLNotesGrabberCORE Console.SetOut(dualLogger); } - List contains = settings.GetValue("ContainsList").Split(',').ToList(); + string? containsListSetting = settings.GetValue("ContainsList"); + if (string.IsNullOrEmpty(containsListSetting)) + throw new InvalidOperationException("ContainsList is not configured in appsettings.json"); + List contains = containsListSetting.Split(',').ToList(); bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false); if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB @@ -175,6 +177,12 @@ namespace URLNotesGrabberCORE break; case "--parse": + if (args.Length < 2) + { + Console.WriteLine("Usage: --parse "); + exitCode = 2; + break; + } string blogNameToParse = args[1]; int postsAdded = 0; try @@ -283,7 +291,7 @@ namespace URLNotesGrabberCORE case "--blogsO": //collect notes from all posts int from = 1, to = 999999, top = 100; - if (args[1] is not null && args[2] is not null && args[3] is not null) + if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null) { from = int.Parse(args[1]); to = int.Parse(args[2]); @@ -293,6 +301,7 @@ namespace URLNotesGrabberCORE { Console.WriteLine("--Expected FROM TO--"); exitCode = 2; + break; } WriteBlogsToFile(settings.GetValue("PathOutputBlogs"), false, from, to, top); break; @@ -300,7 +309,7 @@ namespace URLNotesGrabberCORE case "--bop": //collect notes from all posts from = 1; to = 999999; top = 100; - if (args[1] is not null && args[2] is not null && args[3] is not null) + if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null) { from = int.Parse(args[1]); to = int.Parse(args[2]); @@ -310,6 +319,7 @@ namespace URLNotesGrabberCORE { Console.WriteLine("--Expected FROM TO--"); exitCode = 2; + break; } WriteBlogsToFileAll(settings.GetValue("PathOutputBlogs"), false, from, to, top); break;