From de96c7c1c5a9a191c84599d2974599cd88cd8486 Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 3 May 2026 08:41:21 -0500 Subject: [PATCH] Updated db location logic --- URLNotesGrabberCORE/DataAccess.cs | 211 ++++++++++++++++++++------- URLNotesGrabberCORE/appsettings.json | 2 +- 2 files changed, 158 insertions(+), 55 deletions(-) diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 0c9f9c5..4c33938 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -73,6 +73,7 @@ namespace URLNotesGrabberCORE private static HashSet? _postIdsToExclude; private static readonly object _importPragmaLock = new object(); private static Tuple? _savedImportPragmas; + private static string? _cachedDbPath; static DataAccess() { @@ -95,6 +96,24 @@ namespace URLNotesGrabberCORE } } + private static string GetDefaultDbPath() + { + if (_cachedDbPath != null) + return _cachedDbPath; + + string? configPath = _configuration?["appSettings:PathDB"]; + if (!string.IsNullOrWhiteSpace(configPath)) + { + _cachedDbPath = configPath; + } + else + { + // Fallback to 3 levels up from the executable directory + _cachedDbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "TL.db"); + } + return _cachedDbPath; + } + public static string Q(string input) { return "'" + input.Replace("'", "''") + "'"; @@ -108,8 +127,9 @@ namespace URLNotesGrabberCORE return dateTime; } - public static void EnableImportModePragmas(string DBPath = @"TL.db") + public static void EnableImportModePragmas(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); lock (_importPragmaLock) { if (_savedImportPragmas != null) @@ -158,8 +178,9 @@ namespace URLNotesGrabberCORE } } - public static void RestoreImportModePragmas(string DBPath = @"TL.db") + public static void RestoreImportModePragmas(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); lock (_importPragmaLock) { if (_savedImportPragmas == null) @@ -197,8 +218,9 @@ namespace URLNotesGrabberCORE } } - public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db") + public static void EnsureReplyTextColumnExists(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -249,8 +271,9 @@ namespace URLNotesGrabberCORE } } - public static void EnsureBlogsLikesColumnsExist(string DBPath = @"TL.db") + public static void EnsureBlogsLikesColumnsExist(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -299,8 +322,9 @@ namespace URLNotesGrabberCORE } #region Adds - public static void AddBlog(string blogName, bool byLikes = false, string DBPath = @"TL.db") + public static void AddBlog(string blogName, bool byLikes = false, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); if (blogName.Contains("deact")) @@ -335,8 +359,9 @@ namespace URLNotesGrabberCORE } } - public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string DBPath = @"TL.db", string? rootBlogName = null, string? rootURL = null) + public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null) { + DBPath ??= GetDefaultDbPath(); try { AddBlog(blogName, byLikes, DBPath); } catch { } try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { } try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL); } catch { } @@ -438,8 +463,9 @@ namespace URLNotesGrabberCORE } } - public static void AddAPICount(string DBPath = @"TL.db") + public static void AddAPICount(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -467,8 +493,9 @@ namespace URLNotesGrabberCORE } } - public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db") + public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { } @@ -547,8 +574,9 @@ namespace URLNotesGrabberCORE /// /// /// blogName, postID, lastNoteTimestamp, notesGatheredTimestamp - public static List> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db") + public static List> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); @@ -556,37 +584,94 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "SELECT " + - " MAX(Posts.BlogName) as BlogName, " + Environment.NewLine + - " Posts.PostID, " + Environment.NewLine + - " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine + - " Posts.NotesGatheredDatetime, " + Environment.NewLine + - " CNT.CNT " + Environment.NewLine + - "FROM " + Environment.NewLine + - " Posts " + Environment.NewLine + - " LEFT OUTER JOIN " + Environment.NewLine + - " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + - " LEFT OUTER JOIN " + Environment.NewLine + - " ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " + - "WHERE NotFound = 0 " + Environment.NewLine; + string sql; if (withoutNotesOnly) { - sql += " and HasNotesGathered = 0 " + Environment.NewLine; - sql += "OR ( NotFound = 0 AND Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine; - } + string beforeDateFilter = string.Empty; + if (beforeDate.HasValue) + { + long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds(); + beforeDateFilter = $"WHERE (U.NotesGatheredDateTime < {unixTimestamp} OR U.NotesGatheredDateTime IS NULL)" + Environment.NewLine; + } - // Filter by NotesGatheredDateTime if beforeDate is provided - if (beforeDate.HasValue) + sql = "WITH PostsWithCount AS" + Environment.NewLine + + "(" + Environment.NewLine + + " SELECT " + Environment.NewLine + + " P.BlogName," + Environment.NewLine + + " P.PostID," + Environment.NewLine + + " 1925013599 AS LatestNoteTimestamp," + Environment.NewLine + + " P.NotesGatheredDateTime," + Environment.NewLine + + " COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT," + Environment.NewLine + + " P.HasNotesGathered," + Environment.NewLine + + " P.NotFound," + Environment.NewLine + + " P.PostDate" + Environment.NewLine + + " FROM Posts P" + Environment.NewLine + + ")," + Environment.NewLine + + "Unioned AS" + Environment.NewLine + + "(" + Environment.NewLine + + " SELECT " + Environment.NewLine + + " BlogName," + Environment.NewLine + + " PostID," + Environment.NewLine + + " LatestNoteTimestamp," + Environment.NewLine + + " NotesGatheredDateTime," + Environment.NewLine + + " CNT," + Environment.NewLine + + " PostDate" + Environment.NewLine + + " FROM PostsWithCount" + Environment.NewLine + + " WHERE NotFound = 0" + Environment.NewLine + + " AND HasNotesGathered = 0" + Environment.NewLine + + "" + Environment.NewLine + + " UNION " + Environment.NewLine + + "" + Environment.NewLine + + " SELECT " + Environment.NewLine + + " BlogName," + Environment.NewLine + + " PostID," + Environment.NewLine + + " LatestNoteTimestamp," + Environment.NewLine + + " NotesGatheredDateTime," + Environment.NewLine + + " CNT," + Environment.NewLine + + " PostDate" + Environment.NewLine + + " FROM PostsWithCount" + Environment.NewLine + + " WHERE BlogName = 'zomb-eh'" + Environment.NewLine + + " AND NotFound = 0" + Environment.NewLine + + " AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine + + ")" + Environment.NewLine + + "SELECT" + Environment.NewLine + + " U.BlogName," + Environment.NewLine + + " U.PostID," + Environment.NewLine + + " U.LatestNoteTimestamp," + Environment.NewLine + + " U.NotesGatheredDateTime," + Environment.NewLine + + " U.CNT" + Environment.NewLine + + "FROM Unioned U" + Environment.NewLine + + beforeDateFilter + + "ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;" + Environment.NewLine; + } + else { - long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds(); - sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine; - } + sql = "SELECT " + + " MAX(Posts.BlogName) as BlogName, " + Environment.NewLine + + " Posts.PostID, " + Environment.NewLine + + " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine + + " Posts.NotesGatheredDatetime, " + Environment.NewLine + + " CNT.CNT " + Environment.NewLine + + "FROM " + Environment.NewLine + + " Posts " + Environment.NewLine + + " LEFT OUTER JOIN " + Environment.NewLine + + " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + + " LEFT OUTER JOIN " + Environment.NewLine + + " ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " + + "WHERE NotFound = 0 " + Environment.NewLine; - sql += "GROUP BY " + Environment.NewLine + - " Posts.BlogName, Posts.PostID " + Environment.NewLine + - "ORDER BY " + Environment.NewLine + - " notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine; + if (beforeDate.HasValue) + { + long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds(); + sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine; + } + + sql += "GROUP BY " + Environment.NewLine + + " Posts.BlogName, Posts.PostID " + Environment.NewLine + + "ORDER BY " + Environment.NewLine + + " notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine; + } Console.WriteLine(withoutNotesOnly); Console.WriteLine(sql); @@ -677,8 +762,9 @@ namespace URLNotesGrabberCORE return posts; } - public static List> GetReplies(string DBPath = @"TL.db") + public static List> GetReplies(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); @@ -720,8 +806,9 @@ namespace URLNotesGrabberCORE return posts; } - public static List> GetRepliesWithMissingText(string DBPath = @"TL.db", int limit = 50) + public static List> GetRepliesWithMissingText(string? DBPath = null, int limit = 50) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); @@ -770,8 +857,9 @@ namespace URLNotesGrabberCORE return posts; } - public static List> GetRepliesWithFilledText(string DBPath = @"TL.db", int limit = 1) + public static List> GetRepliesWithFilledText(string? DBPath = null, int limit = 1) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); @@ -823,8 +911,9 @@ namespace URLNotesGrabberCORE return posts; } - public static int GetAPICount(string DBPath = @"TL.db") + public static int GetAPICount(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int count = 0; @@ -861,8 +950,9 @@ namespace URLNotesGrabberCORE return count; } - public static List> GetBlogsForLikes(string specificBlog = null, string DBPath = @"TL.db") + public static List> GetBlogsForLikes(string specificBlog = null, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> blogs = new List>(); @@ -904,8 +994,9 @@ namespace URLNotesGrabberCORE return blogs; } - public static List GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db") + public static List GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); @@ -948,8 +1039,9 @@ namespace URLNotesGrabberCORE return blogs; } - public static List GetBlogsAll(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db") + public static List GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); @@ -991,8 +1083,9 @@ namespace URLNotesGrabberCORE } return blogs; } - public static IEnumerable> GetAllPostTextColumns(string DBPath = @"TL.db") + public static IEnumerable> GetAllPostTextColumns(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); try @@ -1035,8 +1128,9 @@ namespace URLNotesGrabberCORE #region Updates - public static void UpdatePostMarkNotesCollected(string blogName, long postID, string DBPath = @"TL.db") + public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -1066,8 +1160,9 @@ namespace URLNotesGrabberCORE } } - public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db") + public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -1098,8 +1193,9 @@ namespace URLNotesGrabberCORE } } - public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db") + public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -1128,8 +1224,9 @@ namespace URLNotesGrabberCORE } } - public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db") + public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { } @@ -1169,8 +1266,9 @@ namespace URLNotesGrabberCORE return false; } - public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string DBPath = @"TL.db", string? rootBlogName = null, string? rootURL = null) + public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -1200,7 +1298,7 @@ namespace URLNotesGrabberCORE sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, "; sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, "; sql += "hasImage = @hasImage, "; - sql += "ByLikes = @byLikes "; + sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) "; sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND ("; sql += "IFNULL(postDate, '') <> @postDate OR "; sql += "IFNULL(reblogURL, '') <> @reblogURL OR "; @@ -1221,7 +1319,7 @@ namespace URLNotesGrabberCORE sql += "IFNULL(answer, '') <> @answer OR "; sql += "IFNULL(title, '') <> @title OR "; sql += "IFNULL(hasImage, 0) <> @hasImage OR "; - sql += "IFNULL(ByLikes, 0) <> @byLikes OR "; + sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR "; sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR "; sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)"; sql += ")"; @@ -1270,8 +1368,9 @@ namespace URLNotesGrabberCORE } } - public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db") + public static void UpdateBlogOutput(string blogName, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -1298,8 +1397,9 @@ namespace URLNotesGrabberCORE } } - public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string DBPath = @"TL.db") + public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { @@ -1324,8 +1424,9 @@ namespace URLNotesGrabberCORE } } - public static int UpdateAPICount(string DBPath = @"TL.db") + public static int UpdateAPICount(string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int APICount = DataAccess.GetAPICount(); @@ -1357,8 +1458,9 @@ namespace URLNotesGrabberCORE return APICount; } - public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string DBPath = @"TL.db") + public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try @@ -1401,8 +1503,9 @@ namespace URLNotesGrabberCORE } } - public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string DBPath = @"TL.db") + public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null) { + DBPath ??= GetDefaultDbPath(); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index b3f3494..e3ea0e9 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -7,7 +7,7 @@ "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt", "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt", "PathOutputUrls": "u:\\jim\\Documents\\Web Copies\\blogs\\GetUrls.txt", - "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db", + "PathDB": "..\\..\\..\\tl.db", "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot", "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218", "EnableFileLogging": false,