jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote master into txt-validation branch

Browse Source
This commit is contained in:
jim
2026-07-16 10:42:28 -05:00
parent 21a5525094 f541ec4260
commit 5361bb78b8
2 changed files with 61 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+47 -148
View File
@@ -141,7 +141,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas != null)
return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -177,10 +177,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
}
finally
{
connection.Close();
}
}
}
@@ -192,7 +188,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas == null)
return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -219,7 +215,6 @@ namespace URLNotesGrabberCORE
finally
{
_savedImportPragmas = null;
connection.Close();
}
}
}
@@ -254,7 +249,7 @@ namespace URLNotesGrabberCORE
public static void EnsureReplyTextColumnExists(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -298,16 +293,12 @@ namespace URLNotesGrabberCORE
// Breakpoint here
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
}
finally
{
connection.Close();
}
}
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -393,10 +384,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}");
}
finally
{
connection.Close();
}
}
#region Adds
@@ -520,8 +507,12 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateCommand.Parameters.AddWithValue("@blogName", blogName);
updateCommand.Parameters.AddWithValue("@postID", postID);
updateCommand.ExecuteNonQuery();
}
@@ -567,7 +558,7 @@ namespace URLNotesGrabberCORE
public static void AddAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -588,10 +579,6 @@ namespace URLNotesGrabberCORE
// Breakpoint here
//Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
@@ -600,7 +587,7 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { }
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -659,10 +646,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine("^^^^^ - SHORTCUT");
}
}
finally
{
connection2.Close();
}
return false;
}
#endregion Adds
@@ -678,7 +661,7 @@ namespace URLNotesGrabberCORE
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
try
@@ -856,17 +839,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try
@@ -900,17 +879,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try
@@ -951,17 +926,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
try
@@ -1013,17 +984,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
}
finally
{
connection.Close();
}
return posts;
}
public static int GetAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int count = 0;
try { AddAPICount(); } catch { }
@@ -1052,17 +1019,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return count;
}
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
try
@@ -1138,17 +1101,13 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error fetching blogs for likes: {ex.Message}");
}
finally
{
connection.Close();
}
return blogs;
}
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
try
@@ -1183,17 +1142,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return blogs;
}
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
try
@@ -1228,51 +1183,39 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return blogs;
}
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
try
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
using (SQLiteDataReader reader = command.ExecuteReader())
{
using (SQLiteDataReader reader = command.ExecuteReader())
while (reader.Read())
{
while (reader.Read())
List<string> rowTexts = new List<string>();
for (int i = 0; i < reader.FieldCount; i++)
{
List<string> rowTexts = new List<string>();
for (int i = 0; i < reader.FieldCount; i++)
if (!reader.IsDBNull(i))
{
if (!reader.IsDBNull(i))
var val = reader.GetValue(i);
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
{
var val = reader.GetValue(i);
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
{
rowTexts.Add(str);
}
rowTexts.Add(str);
}
}
if (rowTexts.Count > 0)
{
yield return rowTexts;
}
}
if (rowTexts.Count > 0)
{
yield return rowTexts;
}
}
}
}
finally
{
connection.Close();
connection.Dispose();
}
}
#endregion Gets
@@ -1282,7 +1225,7 @@ namespace URLNotesGrabberCORE
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1305,16 +1248,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1338,10 +1277,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
@@ -1453,7 +1388,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1480,10 +1415,6 @@ namespace URLNotesGrabberCORE
return true;
}
}
finally
{
connection.Close();
}
return false;
}
@@ -1599,7 +1530,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogOutput(string blogName, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1619,16 +1550,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -1646,10 +1573,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error updating blog likes status: {ex.Message}");
}
finally
{
connection.Close();
}
}
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
@@ -1659,7 +1582,7 @@ namespace URLNotesGrabberCORE
{
if (newestTimestamp <= 0) return;
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -1679,10 +1602,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
}
finally
{
connection.Close();
}
}
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
@@ -1690,7 +1609,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -1713,16 +1632,12 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
}
finally
{
connection.Close();
}
}
public static int UpdateAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int APICount = DataAccess.GetAPICount();
APICount++;
@@ -1745,10 +1660,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return APICount;
}
@@ -1756,7 +1667,7 @@ namespace URLNotesGrabberCORE
public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int rowsAffected = 0;
// A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
@@ -1799,17 +1710,13 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
}
finally
{
connection.Close();
}
return rowsAffected;
}
public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1845,10 +1752,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
return 0;
}
finally
{
connection.Close();
}
}
#endregion Updates
@@ -1859,7 +1762,7 @@ namespace URLNotesGrabberCORE
public static void EnsureTTFileHelperColumnsExist(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1905,10 +1808,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}");
}
finally
{
connection.Close();
}
}
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
@@ -2729,7 +2628,7 @@ namespace URLNotesGrabberCORE
{
minRetrySeconds = 0;
if (!_usePool || _overrideKey != null) return false;
if (_keys.Count <= 1) return true;
if (_keys.Count == 0) return false;
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
bool all = true;