4 Commits
Author SHA1 Message Date
jim 8d6b9212c1 Remove binaries, batch script; add DB schema verifier
Removed outdated binary files and the `run_500_times.bat` script, which automated repetitive runs of `URLNotesGrabberCORE`. The batch script is no longer needed or has been replaced.

Added `verify-db-schema.sql`, a new script to verify and align the SQLite database schema (`TL.db`) with the application's expected schema. The script includes:
- A verification section to identify missing/extra columns or tables.
- An optional fix section with `ALTER TABLE` statements to add missing columns.

The SQL script ensures database compatibility while preserving data integrity and avoiding destructive operations.
2026-06-03 15:54:22 -05:00
jimandClaude Opus 4.8 18f172fe96 Make -collect 0 a resumable, single-pass full re-check
Mode 0 (full re-check) previously reset its cutoff to now on every
launch, so an interrupted run restarted from scratch, and a post that
kept returning a non-Success/non-NotFound status could loop forever.

- Add single-row CollectRunState table + accessors (EnsureCollectRunStateTableExists,
  GetCollectRunState, BeginCollectRun, CompleteCollectRun) mirroring the
  ApiKeyPoolMeta pattern, to persist a frozen run cutoff and completion flag.
- -collect 0 with no explicit date is now a managed run: resume against the
  stored cutoff if a run is in progress, else start a new run; mark complete
  when the pass finishes so the next launch starts fresh. Explicit-date and
  mode 1 behavior unchanged.
- CollectNotes makes a single attempt pass via an in-process attempted set;
  FAILURE/UNKNOWN are logged once, TooManyRequests/no-lease aborts without
  completing so a later launch resumes.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-06-03 15:48:26 -05:00
jimandClaude Opus 4.8 b576a9cdf3 fix: make -revert scan the PathInput tree like the no-parameter run
-revert was DB-driven, searching each blog's Blogs.TTFolderPath
non-recursively for *.bak. That tree differs from the no-parameter run,
which recursively walks PathInput. Rewrite RevertMode to recursively walk
PathInput (filesystem-only, no DB), with the optional [blogname] argument
now filtering by path substring. Restore mechanics unchanged.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-05-28 16:30:26 -05:00
jimandClaude Opus 4.8 5973920894 feat: add -revert mode to restore *.bak back to *.txt
Inverse of -output. For each blog with a TTFolderPath, restores every
*.bak over its *.txt, first preserving the current *.txt as the
next-free *.bkN, then consuming the *.bak. Confirms before running and
supports an optional single-blog filter.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-05-28 16:11:35 -05:00
9 changed files with 438 additions and 57 deletions
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
+63
View File
@@ -1342,6 +1342,69 @@ namespace URLNotesGrabberCORE
} }
} }
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
// Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the
// same cutoff and lets a completed run stop instead of restarting on the next launch.
public static void EnsureCollectRunStateTableExists(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" +
"Id INTEGER PRIMARY KEY CHECK (Id = 1), " +
"RunCutoff INTEGER DEFAULT 0, " +
"RunComplete INTEGER DEFAULT 1, " +
"RunStarted TEXT, " +
"RunCompletedAt TEXT)";
using SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
// Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet.
public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection);
using SQLiteDataReader reader = command.ExecuteReader();
if (reader.Read())
{
long cutoff = Convert.ToInt64(reader.GetValue(0));
bool complete = Convert.ToInt64(reader.GetValue(1)) != 0;
return (cutoff, complete);
}
return null;
}
// Start (or restart) a managed run: freeze the cutoff and mark the run in progress.
public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " +
"VALUES (1, @cutoff, 0, @started, NULL) " +
"ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL";
using SQLiteCommand command = new SQLiteCommand(sql, connection);
command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds);
command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
// Mark the active managed run complete so the next launch starts fresh instead of resuming.
public static void CompleteCollectRun(string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open();
string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1";
using SQLiteCommand command = new SQLiteCommand(sql, connection);
command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null) public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
+66 -5
View File
@@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file"); Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime"); Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file "); Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -185,6 +185,8 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath"); Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
Console.WriteLine("-revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory"); Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)"); Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
@@ -229,6 +231,7 @@ namespace URLNotesGrabberCORE
case "-collect": //collect notes from all posts case "-collect": //collect notes from all posts
bool withoutNotesOnly = true; bool withoutNotesOnly = true;
DateTime? beforeDate = DateTime.Now; DateTime? beforeDate = DateTime.Now;
bool explicitDateSupplied = false;
if (args.Length < 2) if (args.Length < 2)
{ {
@@ -259,6 +262,7 @@ namespace URLNotesGrabberCORE
if (DateTime.TryParse(args[2], out DateTime parsedDate)) if (DateTime.TryParse(args[2], out DateTime parsedDate))
{ {
beforeDate = parsedDate; beforeDate = parsedDate;
explicitDateSupplied = true;
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}"); Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
} }
else else
@@ -268,7 +272,29 @@ namespace URLNotesGrabberCORE
} }
} }
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult(); // Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
bool managedCollectRun = false;
if (!withoutNotesOnly && !explicitDateSupplied)
{
DataAccess.EnsureCollectRunStateTableExists();
var runState = DataAccess.GetCollectRunState();
if (runState != null && !runState.Value.complete)
{
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
}
else
{
beforeDate = DateTime.Now;
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
}
managedCollectRun = true;
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
break; break;
case "-blogsR": //collect notes from all posts case "-blogsR": //collect notes from all posts
@@ -329,6 +355,10 @@ namespace URLNotesGrabberCORE
OutputMode.Run(config); OutputMode.Run(config);
break; break;
case "-revert":
RevertMode.Run(config, args.Length > 1 ? args[1] : null);
break;
case "-correct": case "-correct":
{ {
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase)); bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
@@ -1182,10 +1212,15 @@ if (shouldInsert)
return "UNKNOWN"; return "UNKNOWN";
} }
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null) static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
{ {
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate); List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{ {
PermitLimit = 300, PermitLimit = 300,
@@ -1202,9 +1237,17 @@ if (shouldInsert)
{ {
while (posts.Count > 0) while (posts.Count > 0)
{ {
// First post not yet attempted this process. If all remaining have been attempted, the pass
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
if (post == null)
{
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
break;
}
ApiKeyPool.SleepUntilAnyAvailable(30); ApiKeyPool.SleepUntilAnyAvailable(30);
var post = posts[0]; // Process the first post in the list
string status; string status;
using RateLimitLease lease = limiter.AttemptAcquire(1); using RateLimitLease lease = limiter.AttemptAcquire(1);
@@ -1216,20 +1259,31 @@ if (shouldInsert)
else else
{ {
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return; return; // throttle: abort without completing the run so a later launch resumes
} }
if (status == "Success") if (status == "Success")
{ {
attempted.Add((post.Item1, post.Item2));
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
} }
else if (status == "NotFound") else if (status == "NotFound")
{ {
attempted.Add((post.Item1, post.Item2));
Console.WriteLine("GrabNotes Result: NotFound"); Console.WriteLine("GrabNotes Result: NotFound");
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
} }
else if (status == "TooManyRequests")
{
// Throttle, not a real per-post failure: don't consume this post's single attempt.
// Abort the pass without completing so a later launch resumes against the same cutoff.
Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
return;
}
else else
{ {
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
attempted.Add((post.Item1, post.Item2));
Console.WriteLine("GrabNotes Result: " + status); Console.WriteLine("GrabNotes Result: " + status);
} }
@@ -1237,6 +1291,13 @@ if (shouldInsert)
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate); posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
} }
} }
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
if (managedRun)
{
DataAccess.CompleteCollectRun();
Console.WriteLine("Full re-check run complete.");
}
} }
catch (Exception ex) catch (Exception ex)
{ {
+110
View File
@@ -0,0 +1,110 @@
using Microsoft.Extensions.Configuration;
namespace URLNotesGrabberCORE
{
// Inverse of OutputMode. Recursively walks the PathInput tree (the same directory tree the
// no-parameter run uses) and restores every *.bak back to its *.txt, first preserving the
// current *.txt as the next-free *.bkN. Consumes the *.bak (File.Move). Filesystem-only;
// does not read the DB. An optional blogname argument filters by path substring.
public static class RevertMode
{
public static int Run(IConfiguration config, string? blogFilter = null)
{
string? root = config["appSettings:PathInput"];
if (string.IsNullOrWhiteSpace(root) || !Directory.Exists(root))
{
Console.WriteLine($"PathInput is not set or does not exist: '{root}'. Nothing to revert.");
return 0;
}
Console.WriteLine($"Searching for .bak files under: {root}");
// Recursively collect every *.bak, optionally filtered by path substring (blogname).
var bakFiles = EnumerateBakFiles(root)
.Where(f => string.IsNullOrWhiteSpace(blogFilter)
|| f.IndexOf(blogFilter, StringComparison.OrdinalIgnoreCase) >= 0)
.ToList();
if (bakFiles.Count == 0)
{
Console.WriteLine("No .bak files found. Nothing to revert.");
return 0;
}
Console.Write($"WARNING: This will restore {bakFiles.Count} .bak file(s) over their .txt files. " +
$"Current .txt files are preserved as the next-free .bkN. Continue? (yes/no): ");
string? response = Console.ReadLine();
if (string.IsNullOrWhiteSpace(response) || !response.Equals("yes", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine("Operation cancelled.");
return 0;
}
int restored = 0, backedUp = 0;
foreach (var bakFile in bakFiles)
{
try
{
string txtPath = Path.ChangeExtension(bakFile, ".txt");
if (File.Exists(txtPath))
{
string bkPath = NextFreeBkPath(txtPath);
File.Move(txtPath, bkPath);
backedUp++;
Console.WriteLine($" Backed up {Path.GetFileName(txtPath)} -> {Path.GetFileName(bkPath)}");
}
File.Move(bakFile, txtPath);
restored++;
Console.WriteLine($" Restored {bakFile} -> {Path.GetFileName(txtPath)}");
}
catch (Exception ex)
{
Console.WriteLine($" Error reverting {bakFile}: {ex.Message}");
}
}
Console.WriteLine($"\nRevert mode complete. Restored {restored} file(s); backed up {backedUp} current .txt file(s).");
return 0;
}
// Recursively yields every *.bak path under root. Per-directory try/catch so an
// inaccessible folder doesn't abort the whole walk (mirrors TraverseDirectory).
private static IEnumerable<string> EnumerateBakFiles(string path)
{
string[] subDirs;
try { subDirs = Directory.GetDirectories(path); }
catch (Exception ex)
{
Console.WriteLine($" Skipping '{path}': {ex.Message}");
yield break;
}
foreach (var dir in subDirs)
foreach (var bak in EnumerateBakFiles(dir))
yield return bak;
string[] bakFiles;
try { bakFiles = Directory.GetFiles(path, "*.bak"); }
catch (Exception ex)
{
Console.WriteLine($" Skipping files in '{path}': {ex.Message}");
yield break;
}
foreach (var bak in bakFiles)
yield return bak;
}
// Returns the lowest unused .bkN path for a given .txt file (.bk1, .bk2, ...).
private static string NextFreeBkPath(string txtFile)
{
for (int n = 1; ; n++)
{
string candidate = Path.ChangeExtension(txtFile, $".bk{n}");
if (!File.Exists(candidate)) return candidate;
}
}
}
}
-52
View File
@@ -1,52 +0,0 @@
@echo off
REM Batch file to run URLNotesGrabberCORE 500 times in a loop
setlocal enabledelayedexpansion
REM Set the path to the executable
REM Update this path if your executable is in a different location
set APP_PATH=URLNotesGrabberCORE.exe
REM Check if the executable exists
if not exist "%APP_PATH%" (
echo Error: %APP_PATH% not found in the current directory.
echo Please ensure the executable is in the same directory as this batch file,
echo or update the APP_PATH variable with the correct path.
pause
exit /b 1
)
REM Loop counter
set ITERATIONS=500
set COUNTER=0
echo Starting to run %APP_PATH% %ITERATIONS% times...
echo.
:LOOP
set /a COUNTER+=1
echo [%COUNTER%/%ITERATIONS%] Running iteration %COUNTER%...
echo Started at: %date% %time%
REM Run the application with -replies option
call "%APP_PATH%" -replies
REM Check if the application ran successfully
if errorlevel 1 (
echo Warning: Application exited with error code !ERRORLEVEL! on iteration %COUNTER%
) else (
echo Iteration %COUNTER% completed successfully.
)
echo Completed at: %date% %time%
echo.
REM Check if we've reached 500 iterations
if %COUNTER% lss %ITERATIONS% (
goto LOOP
)
echo.
echo Completed all %ITERATIONS% iterations!
echo.
pause
+199
View File
@@ -0,0 +1,199 @@
-- ============================================================================
-- verify-db-schema.sql
--
-- Purpose: Verify that a TL.db (e.g. a restored backup) has every column the
-- current URLNotesGrabberCORE code expects. The app has NO startup
-- migration: missing columns only get added when specific modes run,
-- and a referenced-but-missing column causes a "no such column" crash.
--
-- How to use (DB Browser for SQLite):
-- 1. File > Open Database -> pick the restored backup.
-- 2. Execute SQL tab. Run SECTION 1 (it is read-only).
-- * Zero rows from every query = schema is fully aligned, you're done.
-- * Rows in "MISSING COLUMNS" = copy the run_this_to_fix text.
-- 3. If columns are missing: KEEP A COPY OF THE BACKUP FIRST, then go to
-- SECTION 2, uncomment ONLY the ALTER lines that match the report, and run.
-- 4. Re-run SECTION 1 to confirm zero rows.
--
-- This script never UPDATEs/DELETEs/DROPs. In particular it deliberately does
-- NOT replicate the likes-reset that the app's -likes migration performs
-- (DataAccess.cs:375), so existing likes high-water marks are preserved.
-- ============================================================================
-- ============================================================================
-- SECTION 1 -- VERIFICATION (read-only)
-- ============================================================================
-- Expected schema for the current code version.
-- alter_stmt is a runnable ALTER for additively-fixable columns; for base
-- columns it is a 'MANUAL REVIEW' note (a missing base column means the backup
-- predates the table's creation or is damaged -- do not blindly auto-add).
WITH expected(tbl, col, alter_stmt) AS (
VALUES
-- Posts (base columns: manual review if missing)
('Posts','BlogName', 'MANUAL REVIEW - base/PK column missing'),
('Posts','PostID', 'MANUAL REVIEW - base/PK column missing'),
('Posts','HasNotesGathered', 'MANUAL REVIEW - base column missing'),
('Posts','reblogURL', 'MANUAL REVIEW - base column missing'),
('Posts','NotFound', 'MANUAL REVIEW - base column missing'),
('Posts','PostDate', 'MANUAL REVIEW - base column missing'),
('Posts','NotesGatheredDateTime', 'MANUAL REVIEW - base column missing'),
('Posts','HasImage', 'MANUAL REVIEW - base column missing'),
('Posts','PostURL', 'MANUAL REVIEW - base column missing'),
('Posts','Slug', 'MANUAL REVIEW - base column missing'),
('Posts','ReblogKey', 'MANUAL REVIEW - base column missing'),
('Posts','ReblogName', 'MANUAL REVIEW - base column missing'),
('Posts','Summary', 'MANUAL REVIEW - base column missing'),
('Posts','Quote', 'MANUAL REVIEW - base column missing'),
('Posts','Body', 'MANUAL REVIEW - base column missing'),
('Posts','Tags', 'MANUAL REVIEW - base column missing'),
('Posts','Link', 'MANUAL REVIEW - base column missing'),
('Posts','PhotoURL', 'MANUAL REVIEW - base column missing'),
('Posts','PhotoCaption', 'MANUAL REVIEW - base column missing'),
('Posts','DownloadedFiles', 'MANUAL REVIEW - base column missing'),
('Posts','AudioCaption', 'MANUAL REVIEW - base column missing'),
('Posts','Question', 'MANUAL REVIEW - base column missing'),
('Posts','Answer', 'MANUAL REVIEW - base column missing'),
('Posts','Title', 'MANUAL REVIEW - base column missing'),
('Posts','ByLikes', 'MANUAL REVIEW - base column missing'),
('Posts','RootBlogName', 'MANUAL REVIEW - base column missing'),
('Posts','RootURL', 'MANUAL REVIEW - base column missing'),
('Posts','DateModified', 'MANUAL REVIEW - base column missing'),
('Posts','DateCreated', 'MANUAL REVIEW - base column missing'),
-- Posts (additive migration column, auto-fixable)
('Posts','PostType', 'ALTER TABLE Posts ADD COLUMN PostType TEXT;'),
-- Blogs (base columns: manual review if missing)
('Blogs','BlogName', 'MANUAL REVIEW - base/PK column missing'),
('Blogs','HasBeenOutput', 'MANUAL REVIEW - base column missing'),
('Blogs','IsActive', 'MANUAL REVIEW - base column missing'),
('Blogs','DateAdded', 'MANUAL REVIEW - base column missing'),
('Blogs','ByLikes', 'MANUAL REVIEW - base column missing'),
('Blogs','DateModified', 'MANUAL REVIEW - base column missing'),
('Blogs','DateCreated', 'MANUAL REVIEW - base column missing'),
-- Blogs (additive migration columns, auto-fixable)
('Blogs','LikesPulled', 'ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;'),
('Blogs','LikesCursor', 'ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;'),
('Blogs','LikesNewestTimestamp', 'ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;'),
('Blogs','LikesLastRefreshed', 'ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;'),
('Blogs','LikesLastNewCount', 'ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;'),
('Blogs','TTFolderPath', 'ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;'),
-- Notes (base columns: manual review if missing)
('Notes','RootBlogName', 'MANUAL REVIEW - base/PK column missing'),
('Notes','PostID', 'MANUAL REVIEW - base/PK column missing'),
('Notes','NoteBlogName', 'MANUAL REVIEW - base/PK column missing'),
('Notes','TimeStamp', 'MANUAL REVIEW - base/PK column missing'),
('Notes','Type', 'MANUAL REVIEW - base/PK column missing'),
('Notes','DatetimeCrawled', 'MANUAL REVIEW - base column missing'),
('Notes','DateModified', 'MANUAL REVIEW - base column missing'),
('Notes','DateCreated', 'MANUAL REVIEW - base column missing'),
-- Notes (additive migration column, auto-fixable)
('Notes','replyText', 'ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT ''.'';'),
-- DailyAPICount (base columns)
('DailyAPICount','Date', 'MANUAL REVIEW - base/PK column missing'),
('DailyAPICount','APICount', 'MANUAL REVIEW - base column missing'),
-- ApiKeyPoolState (created at runtime by EnsureApiKeyPoolTables; auto-fixable by re-running app, but safe to add)
('ApiKeyPoolState','KeyName', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
('ApiKeyPoolState','RetryUntil', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
('ApiKeyPoolMeta','Id', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
('ApiKeyPoolMeta','LastIndex', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables')
),
actual(tbl, col) AS (
SELECT 'Posts', name FROM pragma_table_info('Posts')
UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs')
UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes')
UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount')
UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState')
UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta')
)
-- 1a. MISSING COLUMNS: columns the code needs that the DB does not have.
-- Zero rows = good. Otherwise copy run_this_to_fix into SECTION 2.
SELECT
e.tbl AS table_name,
e.col AS missing_column,
e.alter_stmt AS run_this_to_fix
FROM expected e
LEFT JOIN actual a
ON a.tbl = e.tbl AND lower(a.col) = lower(e.col)
WHERE a.col IS NULL
ORDER BY (e.alter_stmt LIKE 'ALTER%') DESC, e.tbl, e.col;
-- 1b. MISSING TABLES: expected tables that don't exist at all in this DB.
-- Zero rows = good.
WITH expected_tables(tbl) AS (
VALUES ('Posts'),('Blogs'),('Notes'),('DailyAPICount'),
('ApiKeyPoolState'),('ApiKeyPoolMeta')
)
SELECT et.tbl AS missing_table
FROM expected_tables et
WHERE NOT EXISTS (
SELECT 1 FROM sqlite_master
WHERE type = 'table' AND lower(name) = lower(et.tbl)
)
ORDER BY et.tbl;
-- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
-- list above. Informational only -- e.g. a NEWER backup, or a column this
-- script's expected-list hasn't been updated for. Not an error by itself.
WITH expected(tbl, col) AS (
VALUES
('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
('Posts','NotFound'),('Posts','PostDate'),('Posts','NotesGatheredDateTime'),('Posts','HasImage'),
('Posts','PostURL'),('Posts','Slug'),('Posts','ReblogKey'),('Posts','ReblogName'),('Posts','Summary'),
('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
('Notes','replyText'),
('DailyAPICount','Date'),('DailyAPICount','APICount'),
('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')
),
actual(tbl, col) AS (
SELECT 'Posts', name FROM pragma_table_info('Posts')
UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs')
UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes')
UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount')
UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState')
UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta')
)
SELECT a.tbl AS table_name, a.col AS unexpected_column
FROM actual a
LEFT JOIN expected e
ON e.tbl = a.tbl AND lower(e.col) = lower(a.col)
WHERE e.col IS NULL
ORDER BY a.tbl, a.col;
-- ============================================================================
-- SECTION 2 -- FIX (opt-in, additive only)
--
-- Run ONLY the lines that query 1a flagged with an ALTER statement.
-- KEEP A COPY OF THE BACKUP FIRST. SQLite has no "ADD COLUMN IF NOT EXISTS",
-- so running an ALTER for a column that already exists throws a harmless
-- "duplicate column name" error and changes nothing -- just run the flagged
-- subset. These are the 8 additive migration columns and nothing else; the
-- likes high-water-mark reset is intentionally NOT included.
-- ============================================================================
-- ALTER TABLE Posts ADD COLUMN PostType TEXT;
-- ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;
-- ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;
-- ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;
-- ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;
-- ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;
-- ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;
-- ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';