Compare commits
4
Commits
494d6aa2d4
...
8d6b9212c1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d6b9212c1 | ||
|
|
18f172fe96 | ||
|
|
b576a9cdf3 | ||
|
|
5973920894 |
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1342,6 +1342,69 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
|
||||
// Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the
|
||||
// same cutoff and lets a completed run stop instead of restarting on the next launch.
|
||||
|
||||
public static void EnsureCollectRunStateTableExists(string? DBPath = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
connection.Open();
|
||||
string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" +
|
||||
"Id INTEGER PRIMARY KEY CHECK (Id = 1), " +
|
||||
"RunCutoff INTEGER DEFAULT 0, " +
|
||||
"RunComplete INTEGER DEFAULT 1, " +
|
||||
"RunStarted TEXT, " +
|
||||
"RunCompletedAt TEXT)";
|
||||
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet.
|
||||
public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
connection.Open();
|
||||
using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection);
|
||||
using SQLiteDataReader reader = command.ExecuteReader();
|
||||
if (reader.Read())
|
||||
{
|
||||
long cutoff = Convert.ToInt64(reader.GetValue(0));
|
||||
bool complete = Convert.ToInt64(reader.GetValue(1)) != 0;
|
||||
return (cutoff, complete);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Start (or restart) a managed run: freeze the cutoff and mark the run in progress.
|
||||
public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
connection.Open();
|
||||
string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " +
|
||||
"VALUES (1, @cutoff, 0, @started, NULL) " +
|
||||
"ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL";
|
||||
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||
command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds);
|
||||
command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// Mark the active managed run complete so the next launch starts fresh instead of resuming.
|
||||
public static void CompleteCollectRun(string? DBPath = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
connection.Open();
|
||||
string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1";
|
||||
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||
command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
|
||||
{
|
||||
DBPath ??= GetDefaultDbPath();
|
||||
|
||||
@@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
|
||||
Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
|
||||
|
||||
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
||||
|
||||
@@ -185,6 +185,8 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
|
||||
|
||||
Console.WriteLine("-revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
|
||||
|
||||
Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
||||
|
||||
Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
||||
@@ -229,6 +231,7 @@ namespace URLNotesGrabberCORE
|
||||
case "-collect": //collect notes from all posts
|
||||
bool withoutNotesOnly = true;
|
||||
DateTime? beforeDate = DateTime.Now;
|
||||
bool explicitDateSupplied = false;
|
||||
|
||||
if (args.Length < 2)
|
||||
{
|
||||
@@ -259,6 +262,7 @@ namespace URLNotesGrabberCORE
|
||||
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
||||
{
|
||||
beforeDate = parsedDate;
|
||||
explicitDateSupplied = true;
|
||||
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
||||
}
|
||||
else
|
||||
@@ -268,7 +272,29 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
|
||||
// Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
|
||||
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
|
||||
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
|
||||
bool managedCollectRun = false;
|
||||
if (!withoutNotesOnly && !explicitDateSupplied)
|
||||
{
|
||||
DataAccess.EnsureCollectRunStateTableExists();
|
||||
var runState = DataAccess.GetCollectRunState();
|
||||
if (runState != null && !runState.Value.complete)
|
||||
{
|
||||
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
|
||||
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
|
||||
}
|
||||
else
|
||||
{
|
||||
beforeDate = DateTime.Now;
|
||||
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
|
||||
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
|
||||
}
|
||||
managedCollectRun = true;
|
||||
}
|
||||
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
case "-blogsR": //collect notes from all posts
|
||||
@@ -329,6 +355,10 @@ namespace URLNotesGrabberCORE
|
||||
OutputMode.Run(config);
|
||||
break;
|
||||
|
||||
case "-revert":
|
||||
RevertMode.Run(config, args.Length > 1 ? args[1] : null);
|
||||
break;
|
||||
|
||||
case "-correct":
|
||||
{
|
||||
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
||||
@@ -1182,10 +1212,15 @@ if (shouldInsert)
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
|
||||
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
|
||||
{
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||
|
||||
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
|
||||
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
|
||||
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
|
||||
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 300,
|
||||
@@ -1202,9 +1237,17 @@ if (shouldInsert)
|
||||
{
|
||||
while (posts.Count > 0)
|
||||
{
|
||||
// First post not yet attempted this process. If all remaining have been attempted, the pass
|
||||
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
|
||||
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
|
||||
if (post == null)
|
||||
{
|
||||
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
|
||||
break;
|
||||
}
|
||||
|
||||
ApiKeyPool.SleepUntilAnyAvailable(30);
|
||||
|
||||
var post = posts[0]; // Process the first post in the list
|
||||
string status;
|
||||
|
||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||
@@ -1216,20 +1259,31 @@ if (shouldInsert)
|
||||
else
|
||||
{
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
return;
|
||||
return; // throttle: abort without completing the run so a later launch resumes
|
||||
}
|
||||
|
||||
if (status == "Success")
|
||||
{
|
||||
attempted.Add((post.Item1, post.Item2));
|
||||
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
||||
}
|
||||
else if (status == "NotFound")
|
||||
{
|
||||
attempted.Add((post.Item1, post.Item2));
|
||||
Console.WriteLine("GrabNotes Result: NotFound");
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
}
|
||||
else if (status == "TooManyRequests")
|
||||
{
|
||||
// Throttle, not a real per-post failure: don't consume this post's single attempt.
|
||||
// Abort the pass without completing so a later launch resumes against the same cutoff.
|
||||
Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
|
||||
attempted.Add((post.Item1, post.Item2));
|
||||
Console.WriteLine("GrabNotes Result: " + status);
|
||||
}
|
||||
|
||||
@@ -1237,6 +1291,13 @@ if (shouldInsert)
|
||||
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||
}
|
||||
}
|
||||
|
||||
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
|
||||
if (managedRun)
|
||||
{
|
||||
DataAccess.CompleteCollectRun();
|
||||
Console.WriteLine("Full re-check run complete.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace URLNotesGrabberCORE
|
||||
{
|
||||
// Inverse of OutputMode. Recursively walks the PathInput tree (the same directory tree the
|
||||
// no-parameter run uses) and restores every *.bak back to its *.txt, first preserving the
|
||||
// current *.txt as the next-free *.bkN. Consumes the *.bak (File.Move). Filesystem-only;
|
||||
// does not read the DB. An optional blogname argument filters by path substring.
|
||||
public static class RevertMode
|
||||
{
|
||||
public static int Run(IConfiguration config, string? blogFilter = null)
|
||||
{
|
||||
string? root = config["appSettings:PathInput"];
|
||||
if (string.IsNullOrWhiteSpace(root) || !Directory.Exists(root))
|
||||
{
|
||||
Console.WriteLine($"PathInput is not set or does not exist: '{root}'. Nothing to revert.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Searching for .bak files under: {root}");
|
||||
|
||||
// Recursively collect every *.bak, optionally filtered by path substring (blogname).
|
||||
var bakFiles = EnumerateBakFiles(root)
|
||||
.Where(f => string.IsNullOrWhiteSpace(blogFilter)
|
||||
|| f.IndexOf(blogFilter, StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
.ToList();
|
||||
|
||||
if (bakFiles.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No .bak files found. Nothing to revert.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Console.Write($"WARNING: This will restore {bakFiles.Count} .bak file(s) over their .txt files. " +
|
||||
$"Current .txt files are preserved as the next-free .bkN. Continue? (yes/no): ");
|
||||
string? response = Console.ReadLine();
|
||||
if (string.IsNullOrWhiteSpace(response) || !response.Equals("yes", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Console.WriteLine("Operation cancelled.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int restored = 0, backedUp = 0;
|
||||
foreach (var bakFile in bakFiles)
|
||||
{
|
||||
try
|
||||
{
|
||||
string txtPath = Path.ChangeExtension(bakFile, ".txt");
|
||||
|
||||
if (File.Exists(txtPath))
|
||||
{
|
||||
string bkPath = NextFreeBkPath(txtPath);
|
||||
File.Move(txtPath, bkPath);
|
||||
backedUp++;
|
||||
Console.WriteLine($" Backed up {Path.GetFileName(txtPath)} -> {Path.GetFileName(bkPath)}");
|
||||
}
|
||||
|
||||
File.Move(bakFile, txtPath);
|
||||
restored++;
|
||||
Console.WriteLine($" Restored {bakFile} -> {Path.GetFileName(txtPath)}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($" Error reverting {bakFile}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"\nRevert mode complete. Restored {restored} file(s); backed up {backedUp} current .txt file(s).");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Recursively yields every *.bak path under root. Per-directory try/catch so an
|
||||
// inaccessible folder doesn't abort the whole walk (mirrors TraverseDirectory).
|
||||
private static IEnumerable<string> EnumerateBakFiles(string path)
|
||||
{
|
||||
string[] subDirs;
|
||||
try { subDirs = Directory.GetDirectories(path); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($" Skipping '{path}': {ex.Message}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var dir in subDirs)
|
||||
foreach (var bak in EnumerateBakFiles(dir))
|
||||
yield return bak;
|
||||
|
||||
string[] bakFiles;
|
||||
try { bakFiles = Directory.GetFiles(path, "*.bak"); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($" Skipping files in '{path}': {ex.Message}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var bak in bakFiles)
|
||||
yield return bak;
|
||||
}
|
||||
|
||||
// Returns the lowest unused .bkN path for a given .txt file (.bk1, .bk2, ...).
|
||||
private static string NextFreeBkPath(string txtFile)
|
||||
{
|
||||
for (int n = 1; ; n++)
|
||||
{
|
||||
string candidate = Path.ChangeExtension(txtFile, $".bk{n}");
|
||||
if (!File.Exists(candidate)) return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
@echo off
|
||||
REM Batch file to run URLNotesGrabberCORE 500 times in a loop
|
||||
|
||||
setlocal enabledelayedexpansion
|
||||
|
||||
REM Set the path to the executable
|
||||
REM Update this path if your executable is in a different location
|
||||
set APP_PATH=URLNotesGrabberCORE.exe
|
||||
|
||||
REM Check if the executable exists
|
||||
if not exist "%APP_PATH%" (
|
||||
echo Error: %APP_PATH% not found in the current directory.
|
||||
echo Please ensure the executable is in the same directory as this batch file,
|
||||
echo or update the APP_PATH variable with the correct path.
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
REM Loop counter
|
||||
set ITERATIONS=500
|
||||
set COUNTER=0
|
||||
|
||||
echo Starting to run %APP_PATH% %ITERATIONS% times...
|
||||
echo.
|
||||
|
||||
:LOOP
|
||||
set /a COUNTER+=1
|
||||
echo [%COUNTER%/%ITERATIONS%] Running iteration %COUNTER%...
|
||||
echo Started at: %date% %time%
|
||||
|
||||
REM Run the application with -replies option
|
||||
call "%APP_PATH%" -replies
|
||||
|
||||
REM Check if the application ran successfully
|
||||
if errorlevel 1 (
|
||||
echo Warning: Application exited with error code !ERRORLEVEL! on iteration %COUNTER%
|
||||
) else (
|
||||
echo Iteration %COUNTER% completed successfully.
|
||||
)
|
||||
|
||||
echo Completed at: %date% %time%
|
||||
echo.
|
||||
|
||||
REM Check if we've reached 500 iterations
|
||||
if %COUNTER% lss %ITERATIONS% (
|
||||
goto LOOP
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Completed all %ITERATIONS% iterations!
|
||||
echo.
|
||||
pause
|
||||
@@ -0,0 +1,199 @@
|
||||
-- ============================================================================
|
||||
-- verify-db-schema.sql
|
||||
--
|
||||
-- Purpose: Verify that a TL.db (e.g. a restored backup) has every column the
|
||||
-- current URLNotesGrabberCORE code expects. The app has NO startup
|
||||
-- migration: missing columns only get added when specific modes run,
|
||||
-- and a referenced-but-missing column causes a "no such column" crash.
|
||||
--
|
||||
-- How to use (DB Browser for SQLite):
|
||||
-- 1. File > Open Database -> pick the restored backup.
|
||||
-- 2. Execute SQL tab. Run SECTION 1 (it is read-only).
|
||||
-- * Zero rows from every query = schema is fully aligned, you're done.
|
||||
-- * Rows in "MISSING COLUMNS" = copy the run_this_to_fix text.
|
||||
-- 3. If columns are missing: KEEP A COPY OF THE BACKUP FIRST, then go to
|
||||
-- SECTION 2, uncomment ONLY the ALTER lines that match the report, and run.
|
||||
-- 4. Re-run SECTION 1 to confirm zero rows.
|
||||
--
|
||||
-- This script never UPDATEs/DELETEs/DROPs. In particular it deliberately does
|
||||
-- NOT replicate the likes-reset that the app's -likes migration performs
|
||||
-- (DataAccess.cs:375), so existing likes high-water marks are preserved.
|
||||
-- ============================================================================
|
||||
|
||||
|
||||
-- ============================================================================
|
||||
-- SECTION 1 -- VERIFICATION (read-only)
|
||||
-- ============================================================================
|
||||
|
||||
-- Expected schema for the current code version.
|
||||
-- alter_stmt is a runnable ALTER for additively-fixable columns; for base
|
||||
-- columns it is a 'MANUAL REVIEW' note (a missing base column means the backup
|
||||
-- predates the table's creation or is damaged -- do not blindly auto-add).
|
||||
WITH expected(tbl, col, alter_stmt) AS (
|
||||
VALUES
|
||||
-- Posts (base columns: manual review if missing)
|
||||
('Posts','BlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Posts','PostID', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Posts','HasNotesGathered', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','reblogURL', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','NotFound', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','PostDate', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','NotesGatheredDateTime', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','HasImage', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','PostURL', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Slug', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','ReblogKey', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','ReblogName', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Summary', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Quote', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Body', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Tags', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Link', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','PhotoURL', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','PhotoCaption', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','DownloadedFiles', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','AudioCaption', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Question', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Answer', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','Title', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','ByLikes', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','RootBlogName', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','RootURL', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','DateModified', 'MANUAL REVIEW - base column missing'),
|
||||
('Posts','DateCreated', 'MANUAL REVIEW - base column missing'),
|
||||
-- Posts (additive migration column, auto-fixable)
|
||||
('Posts','PostType', 'ALTER TABLE Posts ADD COLUMN PostType TEXT;'),
|
||||
|
||||
-- Blogs (base columns: manual review if missing)
|
||||
('Blogs','BlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Blogs','HasBeenOutput', 'MANUAL REVIEW - base column missing'),
|
||||
('Blogs','IsActive', 'MANUAL REVIEW - base column missing'),
|
||||
('Blogs','DateAdded', 'MANUAL REVIEW - base column missing'),
|
||||
('Blogs','ByLikes', 'MANUAL REVIEW - base column missing'),
|
||||
('Blogs','DateModified', 'MANUAL REVIEW - base column missing'),
|
||||
('Blogs','DateCreated', 'MANUAL REVIEW - base column missing'),
|
||||
-- Blogs (additive migration columns, auto-fixable)
|
||||
('Blogs','LikesPulled', 'ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;'),
|
||||
('Blogs','LikesCursor', 'ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;'),
|
||||
('Blogs','LikesNewestTimestamp', 'ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;'),
|
||||
('Blogs','LikesLastRefreshed', 'ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;'),
|
||||
('Blogs','LikesLastNewCount', 'ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;'),
|
||||
('Blogs','TTFolderPath', 'ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;'),
|
||||
|
||||
-- Notes (base columns: manual review if missing)
|
||||
('Notes','RootBlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Notes','PostID', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Notes','NoteBlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Notes','TimeStamp', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Notes','Type', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('Notes','DatetimeCrawled', 'MANUAL REVIEW - base column missing'),
|
||||
('Notes','DateModified', 'MANUAL REVIEW - base column missing'),
|
||||
('Notes','DateCreated', 'MANUAL REVIEW - base column missing'),
|
||||
-- Notes (additive migration column, auto-fixable)
|
||||
('Notes','replyText', 'ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT ''.'';'),
|
||||
|
||||
-- DailyAPICount (base columns)
|
||||
('DailyAPICount','Date', 'MANUAL REVIEW - base/PK column missing'),
|
||||
('DailyAPICount','APICount', 'MANUAL REVIEW - base column missing'),
|
||||
|
||||
-- ApiKeyPoolState (created at runtime by EnsureApiKeyPoolTables; auto-fixable by re-running app, but safe to add)
|
||||
('ApiKeyPoolState','KeyName', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
|
||||
('ApiKeyPoolState','RetryUntil', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
|
||||
('ApiKeyPoolMeta','Id', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
|
||||
('ApiKeyPoolMeta','LastIndex', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables')
|
||||
),
|
||||
actual(tbl, col) AS (
|
||||
SELECT 'Posts', name FROM pragma_table_info('Posts')
|
||||
UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs')
|
||||
UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes')
|
||||
UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount')
|
||||
UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState')
|
||||
UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta')
|
||||
)
|
||||
|
||||
-- 1a. MISSING COLUMNS: columns the code needs that the DB does not have.
|
||||
-- Zero rows = good. Otherwise copy run_this_to_fix into SECTION 2.
|
||||
SELECT
|
||||
e.tbl AS table_name,
|
||||
e.col AS missing_column,
|
||||
e.alter_stmt AS run_this_to_fix
|
||||
FROM expected e
|
||||
LEFT JOIN actual a
|
||||
ON a.tbl = e.tbl AND lower(a.col) = lower(e.col)
|
||||
WHERE a.col IS NULL
|
||||
ORDER BY (e.alter_stmt LIKE 'ALTER%') DESC, e.tbl, e.col;
|
||||
|
||||
|
||||
-- 1b. MISSING TABLES: expected tables that don't exist at all in this DB.
|
||||
-- Zero rows = good.
|
||||
WITH expected_tables(tbl) AS (
|
||||
VALUES ('Posts'),('Blogs'),('Notes'),('DailyAPICount'),
|
||||
('ApiKeyPoolState'),('ApiKeyPoolMeta')
|
||||
)
|
||||
SELECT et.tbl AS missing_table
|
||||
FROM expected_tables et
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM sqlite_master
|
||||
WHERE type = 'table' AND lower(name) = lower(et.tbl)
|
||||
)
|
||||
ORDER BY et.tbl;
|
||||
|
||||
|
||||
-- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
|
||||
-- list above. Informational only -- e.g. a NEWER backup, or a column this
|
||||
-- script's expected-list hasn't been updated for. Not an error by itself.
|
||||
WITH expected(tbl, col) AS (
|
||||
VALUES
|
||||
('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
|
||||
('Posts','NotFound'),('Posts','PostDate'),('Posts','NotesGatheredDateTime'),('Posts','HasImage'),
|
||||
('Posts','PostURL'),('Posts','Slug'),('Posts','ReblogKey'),('Posts','ReblogName'),('Posts','Summary'),
|
||||
('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
|
||||
('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
|
||||
('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
|
||||
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
|
||||
('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
|
||||
('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
|
||||
('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
|
||||
('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
|
||||
('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
|
||||
('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
|
||||
('Notes','replyText'),
|
||||
('DailyAPICount','Date'),('DailyAPICount','APICount'),
|
||||
('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
|
||||
('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')
|
||||
),
|
||||
actual(tbl, col) AS (
|
||||
SELECT 'Posts', name FROM pragma_table_info('Posts')
|
||||
UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs')
|
||||
UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes')
|
||||
UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount')
|
||||
UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState')
|
||||
UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta')
|
||||
)
|
||||
SELECT a.tbl AS table_name, a.col AS unexpected_column
|
||||
FROM actual a
|
||||
LEFT JOIN expected e
|
||||
ON e.tbl = a.tbl AND lower(e.col) = lower(a.col)
|
||||
WHERE e.col IS NULL
|
||||
ORDER BY a.tbl, a.col;
|
||||
|
||||
|
||||
-- ============================================================================
|
||||
-- SECTION 2 -- FIX (opt-in, additive only)
|
||||
--
|
||||
-- Run ONLY the lines that query 1a flagged with an ALTER statement.
|
||||
-- KEEP A COPY OF THE BACKUP FIRST. SQLite has no "ADD COLUMN IF NOT EXISTS",
|
||||
-- so running an ALTER for a column that already exists throws a harmless
|
||||
-- "duplicate column name" error and changes nothing -- just run the flagged
|
||||
-- subset. These are the 8 additive migration columns and nothing else; the
|
||||
-- likes high-water-mark reset is intentionally NOT included.
|
||||
-- ============================================================================
|
||||
|
||||
-- ALTER TABLE Posts ADD COLUMN PostType TEXT;
|
||||
-- ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;
|
||||
-- ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;
|
||||
-- ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;
|
||||
-- ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;
|
||||
-- ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;
|
||||
-- ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;
|
||||
-- ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';
|
||||
Reference in New Issue
Block a user