jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote master into txt-validation branch

Browse Source
This commit is contained in:
jim
2026-07-16 10:42:28 -05:00
parent 21a5525094 f541ec4260
commit 5361bb78b8
2 changed files with 61 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+33 -134
View File
@@ -141,7 +141,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas != null)
return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -177,10 +177,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
}
finally
{
connection.Close();
}
}
}
@@ -192,7 +188,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas == null)
return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -219,7 +215,6 @@ namespace URLNotesGrabberCORE
finally
{
_savedImportPragmas = null;
connection.Close();
}
}
}
@@ -254,7 +249,7 @@ namespace URLNotesGrabberCORE
public static void EnsureReplyTextColumnExists(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -298,16 +293,12 @@ namespace URLNotesGrabberCORE
// Breakpoint here
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
}
finally
{
connection.Close();
}
}
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -393,10 +384,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}");
}
finally
{
connection.Close();
}
}
#region Adds
@@ -520,8 +507,12 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateCommand.Parameters.AddWithValue("@blogName", blogName);
updateCommand.Parameters.AddWithValue("@postID", postID);
updateCommand.ExecuteNonQuery();
}
@@ -567,7 +558,7 @@ namespace URLNotesGrabberCORE
public static void AddAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -588,10 +579,6 @@ namespace URLNotesGrabberCORE
// Breakpoint here
//Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
@@ -600,7 +587,7 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { }
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -659,10 +646,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine("^^^^^ - SHORTCUT");
}
}
finally
{
connection2.Close();
}
return false;
}
#endregion Adds
@@ -678,7 +661,7 @@ namespace URLNotesGrabberCORE
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
try
@@ -856,17 +839,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try
@@ -900,17 +879,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try
@@ -951,17 +926,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
try
@@ -1013,17 +984,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
}
finally
{
connection.Close();
}
return posts;
}
public static int GetAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int count = 0;
try { AddAPICount(); } catch { }
@@ -1052,17 +1019,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return count;
}
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
try
@@ -1138,17 +1101,13 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error fetching blogs for likes: {ex.Message}");
}
finally
{
connection.Close();
}
return blogs;
}
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
try
@@ -1183,17 +1142,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return blogs;
}
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
try
@@ -1228,19 +1183,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return blogs;
}
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
try
{
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -1268,12 +1217,6 @@ namespace URLNotesGrabberCORE
}
}
}
finally
{
connection.Close();
connection.Dispose();
}
}
#endregion Gets
#region Updates
@@ -1282,7 +1225,7 @@ namespace URLNotesGrabberCORE
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1305,16 +1248,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1338,10 +1277,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
@@ -1453,7 +1388,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1480,10 +1415,6 @@ namespace URLNotesGrabberCORE
return true;
}
}
finally
{
connection.Close();
}
return false;
}
@@ -1599,7 +1530,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogOutput(string blogName, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1619,16 +1550,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -1646,10 +1573,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error updating blog likes status: {ex.Message}");
}
finally
{
connection.Close();
}
}
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
@@ -1659,7 +1582,7 @@ namespace URLNotesGrabberCORE
{
if (newestTimestamp <= 0) return;
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -1679,10 +1602,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
}
finally
{
connection.Close();
}
}
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
@@ -1690,7 +1609,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
@@ -1713,16 +1632,12 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
}
finally
{
connection.Close();
}
}
public static int UpdateAPICount(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int APICount = DataAccess.GetAPICount();
APICount++;
@@ -1745,10 +1660,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return APICount;
}
@@ -1756,7 +1667,7 @@ namespace URLNotesGrabberCORE
public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int rowsAffected = 0;
// A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
@@ -1799,17 +1710,13 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
}
finally
{
connection.Close();
}
return rowsAffected;
}
public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1845,10 +1752,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
return 0;
}
finally
{
connection.Close();
}
}
#endregion Updates
@@ -1859,7 +1762,7 @@ namespace URLNotesGrabberCORE
public static void EnsureTTFileHelperColumnsExist(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -1905,10 +1808,6 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}");
}
finally
{
connection.Close();
}
}
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
@@ -2729,7 +2628,7 @@ namespace URLNotesGrabberCORE
{
minRetrySeconds = 0;
if (!_usePool || _overrideKey != null) return false;
if (_keys.Count <= 1) return true;
if (_keys.Count == 0) return false;
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
bool all = true;
URLNotesGrabberCORE/Program.cs
+14 -4
View File
@@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration;
using System.Configuration;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Text.RegularExpressions;
namespace URLNotesGrabberCORE
@@ -140,7 +139,10 @@ namespace URLNotesGrabberCORE
Console.SetOut(dualLogger);
}
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
string? containsListSetting = settings.GetValue<string>("ContainsList");
if (string.IsNullOrEmpty(containsListSetting))
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
List<string> contains = containsListSetting.Split(',').ToList();
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
@@ -175,6 +177,12 @@ namespace URLNotesGrabberCORE
break;
case "--parse":
if (args.Length < 2)
{
Console.WriteLine("Usage: --parse <blogname>");
exitCode = 2;
break;
}
string blogNameToParse = args[1];
int postsAdded = 0;
try
@@ -283,7 +291,7 @@ namespace URLNotesGrabberCORE
case "--blogsO": //collect notes from all posts
int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
@@ -293,6 +301,7 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine("--Expected FROM TO--");
exitCode = 2;
break;
}
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
@@ -300,7 +309,7 @@ namespace URLNotesGrabberCORE
case "--bop": //collect notes from all posts
from = 1; to = 999999; top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
@@ -310,6 +319,7 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine("--Expected FROM TO--");
exitCode = 2;
break;
}
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;