jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated db location logic

Browse Source
This commit is contained in:
jim
2026-05-03 08:41:21 -05:00
parent 059c9cd527
commit de96c7c1c5
2 changed files with 158 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+140 -37
View File
@@ -73,6 +73,7 @@ namespace URLNotesGrabberCORE
private static HashSet<long>? _postIdsToExclude;
private static readonly object _importPragmaLock = new object();
private static Tuple<string, string>? _savedImportPragmas;
private static string? _cachedDbPath;
static DataAccess()
{
@@ -95,6 +96,24 @@ namespace URLNotesGrabberCORE
}
}
private static string GetDefaultDbPath()
{
if (_cachedDbPath != null)
return _cachedDbPath;
string? configPath = _configuration?["appSettings:PathDB"];
if (!string.IsNullOrWhiteSpace(configPath))
{
_cachedDbPath = configPath;
}
else
{
// Fallback to 3 levels up from the executable directory
_cachedDbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "TL.db");
}
return _cachedDbPath;
}
public static string Q(string input)
{
return "'" + input.Replace("'", "''") + "'";
@@ -108,8 +127,9 @@ namespace URLNotesGrabberCORE
return dateTime;
}
public static void EnableImportModePragmas(string DBPath = @"TL.db")
public static void EnableImportModePragmas(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
lock (_importPragmaLock)
{
if (_savedImportPragmas != null)
@@ -158,8 +178,9 @@ namespace URLNotesGrabberCORE
}
}
public static void RestoreImportModePragmas(string DBPath = @"TL.db")
public static void RestoreImportModePragmas(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
lock (_importPragmaLock)
{
if (_savedImportPragmas == null)
@@ -197,8 +218,9 @@ namespace URLNotesGrabberCORE
}
}
public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db")
public static void EnsureReplyTextColumnExists(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -249,8 +271,9 @@ namespace URLNotesGrabberCORE
}
}
public static void EnsureBlogsLikesColumnsExist(string DBPath = @"TL.db")
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -299,8 +322,9 @@ namespace URLNotesGrabberCORE
}
#region Adds
public static void AddBlog(string blogName, bool byLikes = false, string DBPath = @"TL.db")
public static void AddBlog(string blogName, bool byLikes = false, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
if (blogName.Contains("deact"))
@@ -335,8 +359,9 @@ namespace URLNotesGrabberCORE
}
}
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string DBPath = @"TL.db", string? rootBlogName = null, string? rootURL = null)
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
{
DBPath ??= GetDefaultDbPath();
try { AddBlog(blogName, byLikes, DBPath); } catch { }
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL); } catch { }
@@ -438,8 +463,9 @@ namespace URLNotesGrabberCORE
}
}
public static void AddAPICount(string DBPath = @"TL.db")
public static void AddAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -467,8 +493,9 @@ namespace URLNotesGrabberCORE
}
}
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { }
@@ -547,8 +574,9 @@ namespace URLNotesGrabberCORE
/// <param name="withoutNotesOnly"></param>
/// <param name="DBPath"></param>
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db")
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
@@ -556,7 +584,70 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "SELECT " +
string sql;
if (withoutNotesOnly)
{
string beforeDateFilter = string.Empty;
if (beforeDate.HasValue)
{
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
beforeDateFilter = $"WHERE (U.NotesGatheredDateTime < {unixTimestamp} OR U.NotesGatheredDateTime IS NULL)" + Environment.NewLine;
}
sql = "WITH PostsWithCount AS" + Environment.NewLine +
"(" + Environment.NewLine +
" SELECT " + Environment.NewLine +
" P.BlogName," + Environment.NewLine +
" P.PostID," + Environment.NewLine +
" 1925013599 AS LatestNoteTimestamp," + Environment.NewLine +
" P.NotesGatheredDateTime," + Environment.NewLine +
" COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT," + Environment.NewLine +
" P.HasNotesGathered," + Environment.NewLine +
" P.NotFound," + Environment.NewLine +
" P.PostDate" + Environment.NewLine +
" FROM Posts P" + Environment.NewLine +
")," + Environment.NewLine +
"Unioned AS" + Environment.NewLine +
"(" + Environment.NewLine +
" SELECT " + Environment.NewLine +
" BlogName," + Environment.NewLine +
" PostID," + Environment.NewLine +
" LatestNoteTimestamp," + Environment.NewLine +
" NotesGatheredDateTime," + Environment.NewLine +
" CNT," + Environment.NewLine +
" PostDate" + Environment.NewLine +
" FROM PostsWithCount" + Environment.NewLine +
" WHERE NotFound = 0" + Environment.NewLine +
" AND HasNotesGathered = 0" + Environment.NewLine +
"" + Environment.NewLine +
" UNION " + Environment.NewLine +
"" + Environment.NewLine +
" SELECT " + Environment.NewLine +
" BlogName," + Environment.NewLine +
" PostID," + Environment.NewLine +
" LatestNoteTimestamp," + Environment.NewLine +
" NotesGatheredDateTime," + Environment.NewLine +
" CNT," + Environment.NewLine +
" PostDate" + Environment.NewLine +
" FROM PostsWithCount" + Environment.NewLine +
" WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
" AND NotFound = 0" + Environment.NewLine +
" AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine +
")" + Environment.NewLine +
"SELECT" + Environment.NewLine +
" U.BlogName," + Environment.NewLine +
" U.PostID," + Environment.NewLine +
" U.LatestNoteTimestamp," + Environment.NewLine +
" U.NotesGatheredDateTime," + Environment.NewLine +
" U.CNT" + Environment.NewLine +
"FROM Unioned U" + Environment.NewLine +
beforeDateFilter +
"ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;" + Environment.NewLine;
}
else
{
sql = "SELECT " +
" MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
" Posts.PostID, " + Environment.NewLine +
" 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
@@ -570,13 +661,6 @@ namespace URLNotesGrabberCORE
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine;
if (withoutNotesOnly)
{
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
sql += "OR ( NotFound = 0 AND Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
}
// Filter by NotesGatheredDateTime if beforeDate is provided
if (beforeDate.HasValue)
{
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
@@ -587,6 +671,7 @@ namespace URLNotesGrabberCORE
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
"ORDER BY " + Environment.NewLine +
" notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine;
}
Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql);
@@ -677,8 +762,9 @@ namespace URLNotesGrabberCORE
return posts;
}
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
@@ -720,8 +806,9 @@ namespace URLNotesGrabberCORE
return posts;
}
public static List<Tuple<string, long>> GetRepliesWithMissingText(string DBPath = @"TL.db", int limit = 50)
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
@@ -770,8 +857,9 @@ namespace URLNotesGrabberCORE
return posts;
}
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string DBPath = @"TL.db", int limit = 1)
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int limit = 1)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
@@ -823,8 +911,9 @@ namespace URLNotesGrabberCORE
return posts;
}
public static int GetAPICount(string DBPath = @"TL.db")
public static int GetAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int count = 0;
@@ -861,8 +950,9 @@ namespace URLNotesGrabberCORE
return count;
}
public static List<Tuple<string, int, long>> GetBlogsForLikes(string specificBlog = null, string DBPath = @"TL.db")
public static List<Tuple<string, int, long>> GetBlogsForLikes(string specificBlog = null, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long>> blogs = new List<Tuple<string, int, long>>();
@@ -904,8 +994,9 @@ namespace URLNotesGrabberCORE
return blogs;
}
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
@@ -948,8 +1039,9 @@ namespace URLNotesGrabberCORE
return blogs;
}
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
@@ -991,8 +1083,9 @@ namespace URLNotesGrabberCORE
}
return blogs;
}
public static IEnumerable<List<string>> GetAllPostTextColumns(string DBPath = @"TL.db")
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
try
@@ -1035,8 +1128,9 @@ namespace URLNotesGrabberCORE
#region Updates
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string DBPath = @"TL.db")
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -1066,8 +1160,9 @@ namespace URLNotesGrabberCORE
}
}
public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db")
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -1098,8 +1193,9 @@ namespace URLNotesGrabberCORE
}
}
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db")
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -1128,8 +1224,9 @@ namespace URLNotesGrabberCORE
}
}
public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { }
@@ -1169,8 +1266,9 @@ namespace URLNotesGrabberCORE
return false;
}
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string DBPath = @"TL.db", string? rootBlogName = null, string? rootURL = null)
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -1200,7 +1298,7 @@ namespace URLNotesGrabberCORE
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
sql += "hasImage = @hasImage, ";
sql += "ByLikes = @byLikes ";
sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) ";
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
sql += "IFNULL(postDate, '') <> @postDate OR ";
sql += "IFNULL(reblogURL, '') <> @reblogURL OR ";
@@ -1221,7 +1319,7 @@ namespace URLNotesGrabberCORE
sql += "IFNULL(answer, '') <> @answer OR ";
sql += "IFNULL(title, '') <> @title OR ";
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
sql += "IFNULL(ByLikes, 0) <> @byLikes OR ";
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)";
sql += ")";
@@ -1270,8 +1368,9 @@ namespace URLNotesGrabberCORE
}
}
public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
public static void UpdateBlogOutput(string blogName, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -1298,8 +1397,9 @@ namespace URLNotesGrabberCORE
}
}
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string DBPath = @"TL.db")
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1324,8 +1424,9 @@ namespace URLNotesGrabberCORE
}
}
public static int UpdateAPICount(string DBPath = @"TL.db")
public static int UpdateAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int APICount = DataAccess.GetAPICount();
@@ -1357,8 +1458,9 @@ namespace URLNotesGrabberCORE
return APICount;
}
public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string DBPath = @"TL.db")
public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -1401,8 +1503,9 @@ namespace URLNotesGrabberCORE
}
}
public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string DBPath = @"TL.db")
public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
URLNotesGrabberCORE/appsettings.json
+1 -1
View File
@@ -7,7 +7,7 @@
"PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
"PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
"PathOutputUrls": "u:\\jim\\Documents\\Web Copies\\blogs\\GetUrls.txt",
"PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
"PathDB": "..\\..\\..\\tl.db",
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot",
"PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218",
"EnableFileLogging": false,