Compare commits
4
Commits
494d6aa2d4
...
8d6b9212c1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d6b9212c1 | ||
|
|
18f172fe96 | ||
|
|
b576a9cdf3 | ||
|
|
5973920894 |
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1342,6 +1342,69 @@ namespace URLNotesGrabberCORE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
|
||||||
|
// Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the
|
||||||
|
// same cutoff and lets a completed run stop instead of restarting on the next launch.
|
||||||
|
|
||||||
|
public static void EnsureCollectRunStateTableExists(string? DBPath = null)
|
||||||
|
{
|
||||||
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
|
connection.Open();
|
||||||
|
string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" +
|
||||||
|
"Id INTEGER PRIMARY KEY CHECK (Id = 1), " +
|
||||||
|
"RunCutoff INTEGER DEFAULT 0, " +
|
||||||
|
"RunComplete INTEGER DEFAULT 1, " +
|
||||||
|
"RunStarted TEXT, " +
|
||||||
|
"RunCompletedAt TEXT)";
|
||||||
|
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||||
|
command.ExecuteNonQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet.
|
||||||
|
public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null)
|
||||||
|
{
|
||||||
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
|
connection.Open();
|
||||||
|
using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection);
|
||||||
|
using SQLiteDataReader reader = command.ExecuteReader();
|
||||||
|
if (reader.Read())
|
||||||
|
{
|
||||||
|
long cutoff = Convert.ToInt64(reader.GetValue(0));
|
||||||
|
bool complete = Convert.ToInt64(reader.GetValue(1)) != 0;
|
||||||
|
return (cutoff, complete);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start (or restart) a managed run: freeze the cutoff and mark the run in progress.
|
||||||
|
public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null)
|
||||||
|
{
|
||||||
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
|
connection.Open();
|
||||||
|
string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " +
|
||||||
|
"VALUES (1, @cutoff, 0, @started, NULL) " +
|
||||||
|
"ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL";
|
||||||
|
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||||
|
command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds);
|
||||||
|
command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
|
command.ExecuteNonQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark the active managed run complete so the next launch starts fresh instead of resuming.
|
||||||
|
public static void CompleteCollectRun(string? DBPath = null)
|
||||||
|
{
|
||||||
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
|
connection.Open();
|
||||||
|
string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1";
|
||||||
|
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||||
|
command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
|
command.ExecuteNonQuery();
|
||||||
|
}
|
||||||
|
|
||||||
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
|
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
|
||||||
{
|
{
|
||||||
DBPath ??= GetDefaultDbPath();
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ namespace URLNotesGrabberCORE
|
|||||||
|
|
||||||
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
||||||
|
|
||||||
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
|
Console.WriteLine("-collect [0|1] [datetime]\t Collect Notes from API. 1=only posts without notes. 0=full re-check of all posts: a single resumable pass (interrupt & relaunch to resume; stops when complete, retrigger for a new pass). Optional datetime overrides the cutoff and runs as a one-off (bypasses resume tracking).");
|
||||||
|
|
||||||
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
||||||
|
|
||||||
@@ -185,6 +185,8 @@ namespace URLNotesGrabberCORE
|
|||||||
|
|
||||||
Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
|
Console.WriteLine("-output\t Export posts from TL.db back to .txt files in each blog's TTFolderPath");
|
||||||
|
|
||||||
|
Console.WriteLine("-revert [blogname]\t Recursively scan the PathInput tree and restore *.bak back to *.txt (current .txt saved as next-free .bkN); optional blogname filters by path substring");
|
||||||
|
|
||||||
Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
Console.WriteLine("-correct [bakPath]\t Dry-run: report multi-line field updates available from a BAK directory");
|
||||||
|
|
||||||
Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
Console.WriteLine("-correct -apply [bakPath]\t Apply BAK-file corrections to matching posts (prompts yes/no)");
|
||||||
@@ -229,6 +231,7 @@ namespace URLNotesGrabberCORE
|
|||||||
case "-collect": //collect notes from all posts
|
case "-collect": //collect notes from all posts
|
||||||
bool withoutNotesOnly = true;
|
bool withoutNotesOnly = true;
|
||||||
DateTime? beforeDate = DateTime.Now;
|
DateTime? beforeDate = DateTime.Now;
|
||||||
|
bool explicitDateSupplied = false;
|
||||||
|
|
||||||
if (args.Length < 2)
|
if (args.Length < 2)
|
||||||
{
|
{
|
||||||
@@ -259,6 +262,7 @@ namespace URLNotesGrabberCORE
|
|||||||
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
||||||
{
|
{
|
||||||
beforeDate = parsedDate;
|
beforeDate = parsedDate;
|
||||||
|
explicitDateSupplied = true;
|
||||||
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -268,7 +272,29 @@ namespace URLNotesGrabberCORE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate).GetAwaiter().GetResult();
|
// Mode 0 (full re-check) with no explicit date is a *managed* run: freeze the cutoff and
|
||||||
|
// persist it so an interrupted run resumes against the same cutoff and a completed run stops
|
||||||
|
// instead of restarting. Mode 1 and explicit-date runs keep their existing behavior.
|
||||||
|
bool managedCollectRun = false;
|
||||||
|
if (!withoutNotesOnly && !explicitDateSupplied)
|
||||||
|
{
|
||||||
|
DataAccess.EnsureCollectRunStateTableExists();
|
||||||
|
var runState = DataAccess.GetCollectRunState();
|
||||||
|
if (runState != null && !runState.Value.complete)
|
||||||
|
{
|
||||||
|
beforeDate = DateTimeOffset.FromUnixTimeSeconds(runState.Value.cutoff).LocalDateTime;
|
||||||
|
Console.WriteLine($"Resuming interrupted full re-check (cutoff = {beforeDate})");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
beforeDate = DateTime.Now;
|
||||||
|
DataAccess.BeginCollectRun(new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds());
|
||||||
|
Console.WriteLine($"Starting new full re-check run (cutoff = {beforeDate})");
|
||||||
|
}
|
||||||
|
managedCollectRun = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate, managedCollectRun).GetAwaiter().GetResult();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "-blogsR": //collect notes from all posts
|
case "-blogsR": //collect notes from all posts
|
||||||
@@ -329,6 +355,10 @@ namespace URLNotesGrabberCORE
|
|||||||
OutputMode.Run(config);
|
OutputMode.Run(config);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case "-revert":
|
||||||
|
RevertMode.Run(config, args.Length > 1 ? args[1] : null);
|
||||||
|
break;
|
||||||
|
|
||||||
case "-correct":
|
case "-correct":
|
||||||
{
|
{
|
||||||
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
bool applyChanges = args.Skip(1).Any(a => string.Equals(a, "-apply", StringComparison.OrdinalIgnoreCase) || string.Equals(a, "--apply", StringComparison.OrdinalIgnoreCase));
|
||||||
@@ -1182,10 +1212,15 @@ if (shouldInsert)
|
|||||||
return "UNKNOWN";
|
return "UNKNOWN";
|
||||||
}
|
}
|
||||||
|
|
||||||
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
|
static async Task CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null, bool managedRun = false)
|
||||||
{
|
{
|
||||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||||
|
|
||||||
|
// Posts attempted (with a definitive, non-throttle result) during *this* process. Guarantees a single
|
||||||
|
// attempt pass: once every remaining post has been attempted, the loop stops instead of spinning on a
|
||||||
|
// post that keeps returning FAILURE/UNKNOWN. Successful/NotFound posts drop out via the DB filter anyway.
|
||||||
|
HashSet<(string, long)> attempted = new HashSet<(string, long)>();
|
||||||
|
|
||||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||||
{
|
{
|
||||||
PermitLimit = 300,
|
PermitLimit = 300,
|
||||||
@@ -1202,9 +1237,17 @@ if (shouldInsert)
|
|||||||
{
|
{
|
||||||
while (posts.Count > 0)
|
while (posts.Count > 0)
|
||||||
{
|
{
|
||||||
|
// First post not yet attempted this process. If all remaining have been attempted, the pass
|
||||||
|
// is done (the stragglers returned FAILURE/UNKNOWN) — stop rather than loop forever.
|
||||||
|
var post = posts.FirstOrDefault(p => !attempted.Contains((p.Item1, p.Item2)));
|
||||||
|
if (post == null)
|
||||||
|
{
|
||||||
|
Console.WriteLine("All remaining posts have been attempted this run; ending pass.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
ApiKeyPool.SleepUntilAnyAvailable(30);
|
ApiKeyPool.SleepUntilAnyAvailable(30);
|
||||||
|
|
||||||
var post = posts[0]; // Process the first post in the list
|
|
||||||
string status;
|
string status;
|
||||||
|
|
||||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||||
@@ -1216,20 +1259,31 @@ if (shouldInsert)
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||||
return;
|
return; // throttle: abort without completing the run so a later launch resumes
|
||||||
}
|
}
|
||||||
|
|
||||||
if (status == "Success")
|
if (status == "Success")
|
||||||
{
|
{
|
||||||
|
attempted.Add((post.Item1, post.Item2));
|
||||||
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
||||||
}
|
}
|
||||||
else if (status == "NotFound")
|
else if (status == "NotFound")
|
||||||
{
|
{
|
||||||
|
attempted.Add((post.Item1, post.Item2));
|
||||||
Console.WriteLine("GrabNotes Result: NotFound");
|
Console.WriteLine("GrabNotes Result: NotFound");
|
||||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||||
}
|
}
|
||||||
|
else if (status == "TooManyRequests")
|
||||||
|
{
|
||||||
|
// Throttle, not a real per-post failure: don't consume this post's single attempt.
|
||||||
|
// Abort the pass without completing so a later launch resumes against the same cutoff.
|
||||||
|
Console.WriteLine("GrabNotes Result: TooManyRequests - pausing run; relaunch to resume.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
// FAILURE / UNKNOWN: count as attempted so the pass can finish instead of retrying forever.
|
||||||
|
attempted.Add((post.Item1, post.Item2));
|
||||||
Console.WriteLine("GrabNotes Result: " + status);
|
Console.WriteLine("GrabNotes Result: " + status);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1237,6 +1291,13 @@ if (shouldInsert)
|
|||||||
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reached only when the pass finished naturally (worklist drained or all stragglers attempted).
|
||||||
|
if (managedRun)
|
||||||
|
{
|
||||||
|
DataAccess.CompleteCollectRun();
|
||||||
|
Console.WriteLine("Full re-check run complete.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
|
||||||
|
namespace URLNotesGrabberCORE
|
||||||
|
{
|
||||||
|
// Inverse of OutputMode. Recursively walks the PathInput tree (the same directory tree the
|
||||||
|
// no-parameter run uses) and restores every *.bak back to its *.txt, first preserving the
|
||||||
|
// current *.txt as the next-free *.bkN. Consumes the *.bak (File.Move). Filesystem-only;
|
||||||
|
// does not read the DB. An optional blogname argument filters by path substring.
|
||||||
|
public static class RevertMode
|
||||||
|
{
|
||||||
|
public static int Run(IConfiguration config, string? blogFilter = null)
|
||||||
|
{
|
||||||
|
string? root = config["appSettings:PathInput"];
|
||||||
|
if (string.IsNullOrWhiteSpace(root) || !Directory.Exists(root))
|
||||||
|
{
|
||||||
|
Console.WriteLine($"PathInput is not set or does not exist: '{root}'. Nothing to revert.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Console.WriteLine($"Searching for .bak files under: {root}");
|
||||||
|
|
||||||
|
// Recursively collect every *.bak, optionally filtered by path substring (blogname).
|
||||||
|
var bakFiles = EnumerateBakFiles(root)
|
||||||
|
.Where(f => string.IsNullOrWhiteSpace(blogFilter)
|
||||||
|
|| f.IndexOf(blogFilter, StringComparison.OrdinalIgnoreCase) >= 0)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (bakFiles.Count == 0)
|
||||||
|
{
|
||||||
|
Console.WriteLine("No .bak files found. Nothing to revert.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Console.Write($"WARNING: This will restore {bakFiles.Count} .bak file(s) over their .txt files. " +
|
||||||
|
$"Current .txt files are preserved as the next-free .bkN. Continue? (yes/no): ");
|
||||||
|
string? response = Console.ReadLine();
|
||||||
|
if (string.IsNullOrWhiteSpace(response) || !response.Equals("yes", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
Console.WriteLine("Operation cancelled.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int restored = 0, backedUp = 0;
|
||||||
|
foreach (var bakFile in bakFiles)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string txtPath = Path.ChangeExtension(bakFile, ".txt");
|
||||||
|
|
||||||
|
if (File.Exists(txtPath))
|
||||||
|
{
|
||||||
|
string bkPath = NextFreeBkPath(txtPath);
|
||||||
|
File.Move(txtPath, bkPath);
|
||||||
|
backedUp++;
|
||||||
|
Console.WriteLine($" Backed up {Path.GetFileName(txtPath)} -> {Path.GetFileName(bkPath)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
File.Move(bakFile, txtPath);
|
||||||
|
restored++;
|
||||||
|
Console.WriteLine($" Restored {bakFile} -> {Path.GetFileName(txtPath)}");
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine($" Error reverting {bakFile}: {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Console.WriteLine($"\nRevert mode complete. Restored {restored} file(s); backed up {backedUp} current .txt file(s).");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively yields every *.bak path under root. Per-directory try/catch so an
|
||||||
|
// inaccessible folder doesn't abort the whole walk (mirrors TraverseDirectory).
|
||||||
|
private static IEnumerable<string> EnumerateBakFiles(string path)
|
||||||
|
{
|
||||||
|
string[] subDirs;
|
||||||
|
try { subDirs = Directory.GetDirectories(path); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine($" Skipping '{path}': {ex.Message}");
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var dir in subDirs)
|
||||||
|
foreach (var bak in EnumerateBakFiles(dir))
|
||||||
|
yield return bak;
|
||||||
|
|
||||||
|
string[] bakFiles;
|
||||||
|
try { bakFiles = Directory.GetFiles(path, "*.bak"); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine($" Skipping files in '{path}': {ex.Message}");
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var bak in bakFiles)
|
||||||
|
yield return bak;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns the lowest unused .bkN path for a given .txt file (.bk1, .bk2, ...).
|
||||||
|
private static string NextFreeBkPath(string txtFile)
|
||||||
|
{
|
||||||
|
for (int n = 1; ; n++)
|
||||||
|
{
|
||||||
|
string candidate = Path.ChangeExtension(txtFile, $".bk{n}");
|
||||||
|
if (!File.Exists(candidate)) return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
@echo off
|
|
||||||
REM Batch file to run URLNotesGrabberCORE 500 times in a loop
|
|
||||||
|
|
||||||
setlocal enabledelayedexpansion
|
|
||||||
|
|
||||||
REM Set the path to the executable
|
|
||||||
REM Update this path if your executable is in a different location
|
|
||||||
set APP_PATH=URLNotesGrabberCORE.exe
|
|
||||||
|
|
||||||
REM Check if the executable exists
|
|
||||||
if not exist "%APP_PATH%" (
|
|
||||||
echo Error: %APP_PATH% not found in the current directory.
|
|
||||||
echo Please ensure the executable is in the same directory as this batch file,
|
|
||||||
echo or update the APP_PATH variable with the correct path.
|
|
||||||
pause
|
|
||||||
exit /b 1
|
|
||||||
)
|
|
||||||
|
|
||||||
REM Loop counter
|
|
||||||
set ITERATIONS=500
|
|
||||||
set COUNTER=0
|
|
||||||
|
|
||||||
echo Starting to run %APP_PATH% %ITERATIONS% times...
|
|
||||||
echo.
|
|
||||||
|
|
||||||
:LOOP
|
|
||||||
set /a COUNTER+=1
|
|
||||||
echo [%COUNTER%/%ITERATIONS%] Running iteration %COUNTER%...
|
|
||||||
echo Started at: %date% %time%
|
|
||||||
|
|
||||||
REM Run the application with -replies option
|
|
||||||
call "%APP_PATH%" -replies
|
|
||||||
|
|
||||||
REM Check if the application ran successfully
|
|
||||||
if errorlevel 1 (
|
|
||||||
echo Warning: Application exited with error code !ERRORLEVEL! on iteration %COUNTER%
|
|
||||||
) else (
|
|
||||||
echo Iteration %COUNTER% completed successfully.
|
|
||||||
)
|
|
||||||
|
|
||||||
echo Completed at: %date% %time%
|
|
||||||
echo.
|
|
||||||
|
|
||||||
REM Check if we've reached 500 iterations
|
|
||||||
if %COUNTER% lss %ITERATIONS% (
|
|
||||||
goto LOOP
|
|
||||||
)
|
|
||||||
|
|
||||||
echo.
|
|
||||||
echo Completed all %ITERATIONS% iterations!
|
|
||||||
echo.
|
|
||||||
pause
|
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
-- ============================================================================
|
||||||
|
-- verify-db-schema.sql
|
||||||
|
--
|
||||||
|
-- Purpose: Verify that a TL.db (e.g. a restored backup) has every column the
|
||||||
|
-- current URLNotesGrabberCORE code expects. The app has NO startup
|
||||||
|
-- migration: missing columns only get added when specific modes run,
|
||||||
|
-- and a referenced-but-missing column causes a "no such column" crash.
|
||||||
|
--
|
||||||
|
-- How to use (DB Browser for SQLite):
|
||||||
|
-- 1. File > Open Database -> pick the restored backup.
|
||||||
|
-- 2. Execute SQL tab. Run SECTION 1 (it is read-only).
|
||||||
|
-- * Zero rows from every query = schema is fully aligned, you're done.
|
||||||
|
-- * Rows in "MISSING COLUMNS" = copy the run_this_to_fix text.
|
||||||
|
-- 3. If columns are missing: KEEP A COPY OF THE BACKUP FIRST, then go to
|
||||||
|
-- SECTION 2, uncomment ONLY the ALTER lines that match the report, and run.
|
||||||
|
-- 4. Re-run SECTION 1 to confirm zero rows.
|
||||||
|
--
|
||||||
|
-- This script never UPDATEs/DELETEs/DROPs. In particular it deliberately does
|
||||||
|
-- NOT replicate the likes-reset that the app's -likes migration performs
|
||||||
|
-- (DataAccess.cs:375), so existing likes high-water marks are preserved.
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- SECTION 1 -- VERIFICATION (read-only)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Expected schema for the current code version.
|
||||||
|
-- alter_stmt is a runnable ALTER for additively-fixable columns; for base
|
||||||
|
-- columns it is a 'MANUAL REVIEW' note (a missing base column means the backup
|
||||||
|
-- predates the table's creation or is damaged -- do not blindly auto-add).
|
||||||
|
WITH expected(tbl, col, alter_stmt) AS (
|
||||||
|
VALUES
|
||||||
|
-- Posts (base columns: manual review if missing)
|
||||||
|
('Posts','BlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Posts','PostID', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Posts','HasNotesGathered', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','reblogURL', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','NotFound', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','PostDate', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','NotesGatheredDateTime', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','HasImage', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','PostURL', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Slug', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','ReblogKey', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','ReblogName', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Summary', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Quote', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Body', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Tags', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Link', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','PhotoURL', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','PhotoCaption', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','DownloadedFiles', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','AudioCaption', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Question', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Answer', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','Title', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','ByLikes', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','RootBlogName', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','RootURL', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','DateModified', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Posts','DateCreated', 'MANUAL REVIEW - base column missing'),
|
||||||
|
-- Posts (additive migration column, auto-fixable)
|
||||||
|
('Posts','PostType', 'ALTER TABLE Posts ADD COLUMN PostType TEXT;'),
|
||||||
|
|
||||||
|
-- Blogs (base columns: manual review if missing)
|
||||||
|
('Blogs','BlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Blogs','HasBeenOutput', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Blogs','IsActive', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Blogs','DateAdded', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Blogs','ByLikes', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Blogs','DateModified', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Blogs','DateCreated', 'MANUAL REVIEW - base column missing'),
|
||||||
|
-- Blogs (additive migration columns, auto-fixable)
|
||||||
|
('Blogs','LikesPulled', 'ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;'),
|
||||||
|
('Blogs','LikesCursor', 'ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;'),
|
||||||
|
('Blogs','LikesNewestTimestamp', 'ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;'),
|
||||||
|
('Blogs','LikesLastRefreshed', 'ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;'),
|
||||||
|
('Blogs','LikesLastNewCount', 'ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;'),
|
||||||
|
('Blogs','TTFolderPath', 'ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;'),
|
||||||
|
|
||||||
|
-- Notes (base columns: manual review if missing)
|
||||||
|
('Notes','RootBlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Notes','PostID', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Notes','NoteBlogName', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Notes','TimeStamp', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Notes','Type', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('Notes','DatetimeCrawled', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Notes','DateModified', 'MANUAL REVIEW - base column missing'),
|
||||||
|
('Notes','DateCreated', 'MANUAL REVIEW - base column missing'),
|
||||||
|
-- Notes (additive migration column, auto-fixable)
|
||||||
|
('Notes','replyText', 'ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT ''.'';'),
|
||||||
|
|
||||||
|
-- DailyAPICount (base columns)
|
||||||
|
('DailyAPICount','Date', 'MANUAL REVIEW - base/PK column missing'),
|
||||||
|
('DailyAPICount','APICount', 'MANUAL REVIEW - base column missing'),
|
||||||
|
|
||||||
|
-- ApiKeyPoolState (created at runtime by EnsureApiKeyPoolTables; auto-fixable by re-running app, but safe to add)
|
||||||
|
('ApiKeyPoolState','KeyName', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
|
||||||
|
('ApiKeyPoolState','RetryUntil', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
|
||||||
|
('ApiKeyPoolMeta','Id', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables'),
|
||||||
|
('ApiKeyPoolMeta','LastIndex', 'MANUAL REVIEW - run app once to auto-create ApiKeyPool tables')
|
||||||
|
),
|
||||||
|
actual(tbl, col) AS (
|
||||||
|
SELECT 'Posts', name FROM pragma_table_info('Posts')
|
||||||
|
UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs')
|
||||||
|
UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes')
|
||||||
|
UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount')
|
||||||
|
UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState')
|
||||||
|
UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta')
|
||||||
|
)
|
||||||
|
|
||||||
|
-- 1a. MISSING COLUMNS: columns the code needs that the DB does not have.
|
||||||
|
-- Zero rows = good. Otherwise copy run_this_to_fix into SECTION 2.
|
||||||
|
SELECT
|
||||||
|
e.tbl AS table_name,
|
||||||
|
e.col AS missing_column,
|
||||||
|
e.alter_stmt AS run_this_to_fix
|
||||||
|
FROM expected e
|
||||||
|
LEFT JOIN actual a
|
||||||
|
ON a.tbl = e.tbl AND lower(a.col) = lower(e.col)
|
||||||
|
WHERE a.col IS NULL
|
||||||
|
ORDER BY (e.alter_stmt LIKE 'ALTER%') DESC, e.tbl, e.col;
|
||||||
|
|
||||||
|
|
||||||
|
-- 1b. MISSING TABLES: expected tables that don't exist at all in this DB.
|
||||||
|
-- Zero rows = good.
|
||||||
|
WITH expected_tables(tbl) AS (
|
||||||
|
VALUES ('Posts'),('Blogs'),('Notes'),('DailyAPICount'),
|
||||||
|
('ApiKeyPoolState'),('ApiKeyPoolMeta')
|
||||||
|
)
|
||||||
|
SELECT et.tbl AS missing_table
|
||||||
|
FROM expected_tables et
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM sqlite_master
|
||||||
|
WHERE type = 'table' AND lower(name) = lower(et.tbl)
|
||||||
|
)
|
||||||
|
ORDER BY et.tbl;
|
||||||
|
|
||||||
|
|
||||||
|
-- 1c. EXTRA / UNEXPECTED COLUMNS: present in the DB but not in the expected
|
||||||
|
-- list above. Informational only -- e.g. a NEWER backup, or a column this
|
||||||
|
-- script's expected-list hasn't been updated for. Not an error by itself.
|
||||||
|
WITH expected(tbl, col) AS (
|
||||||
|
VALUES
|
||||||
|
('Posts','BlogName'),('Posts','PostID'),('Posts','HasNotesGathered'),('Posts','reblogURL'),
|
||||||
|
('Posts','NotFound'),('Posts','PostDate'),('Posts','NotesGatheredDateTime'),('Posts','HasImage'),
|
||||||
|
('Posts','PostURL'),('Posts','Slug'),('Posts','ReblogKey'),('Posts','ReblogName'),('Posts','Summary'),
|
||||||
|
('Posts','Quote'),('Posts','Body'),('Posts','Tags'),('Posts','Link'),('Posts','PhotoURL'),
|
||||||
|
('Posts','PhotoCaption'),('Posts','DownloadedFiles'),('Posts','AudioCaption'),('Posts','Question'),
|
||||||
|
('Posts','Answer'),('Posts','Title'),('Posts','ByLikes'),('Posts','RootBlogName'),('Posts','RootURL'),
|
||||||
|
('Posts','DateModified'),('Posts','DateCreated'),('Posts','PostType'),
|
||||||
|
('Blogs','BlogName'),('Blogs','HasBeenOutput'),('Blogs','IsActive'),('Blogs','DateAdded'),
|
||||||
|
('Blogs','ByLikes'),('Blogs','DateModified'),('Blogs','DateCreated'),('Blogs','LikesPulled'),
|
||||||
|
('Blogs','LikesCursor'),('Blogs','LikesNewestTimestamp'),('Blogs','LikesLastRefreshed'),
|
||||||
|
('Blogs','LikesLastNewCount'),('Blogs','TTFolderPath'),
|
||||||
|
('Notes','RootBlogName'),('Notes','PostID'),('Notes','NoteBlogName'),('Notes','TimeStamp'),
|
||||||
|
('Notes','Type'),('Notes','DatetimeCrawled'),('Notes','DateModified'),('Notes','DateCreated'),
|
||||||
|
('Notes','replyText'),
|
||||||
|
('DailyAPICount','Date'),('DailyAPICount','APICount'),
|
||||||
|
('ApiKeyPoolState','KeyName'),('ApiKeyPoolState','RetryUntil'),
|
||||||
|
('ApiKeyPoolMeta','Id'),('ApiKeyPoolMeta','LastIndex')
|
||||||
|
),
|
||||||
|
actual(tbl, col) AS (
|
||||||
|
SELECT 'Posts', name FROM pragma_table_info('Posts')
|
||||||
|
UNION ALL SELECT 'Blogs', name FROM pragma_table_info('Blogs')
|
||||||
|
UNION ALL SELECT 'Notes', name FROM pragma_table_info('Notes')
|
||||||
|
UNION ALL SELECT 'DailyAPICount', name FROM pragma_table_info('DailyAPICount')
|
||||||
|
UNION ALL SELECT 'ApiKeyPoolState', name FROM pragma_table_info('ApiKeyPoolState')
|
||||||
|
UNION ALL SELECT 'ApiKeyPoolMeta', name FROM pragma_table_info('ApiKeyPoolMeta')
|
||||||
|
)
|
||||||
|
SELECT a.tbl AS table_name, a.col AS unexpected_column
|
||||||
|
FROM actual a
|
||||||
|
LEFT JOIN expected e
|
||||||
|
ON e.tbl = a.tbl AND lower(e.col) = lower(a.col)
|
||||||
|
WHERE e.col IS NULL
|
||||||
|
ORDER BY a.tbl, a.col;
|
||||||
|
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- SECTION 2 -- FIX (opt-in, additive only)
|
||||||
|
--
|
||||||
|
-- Run ONLY the lines that query 1a flagged with an ALTER statement.
|
||||||
|
-- KEEP A COPY OF THE BACKUP FIRST. SQLite has no "ADD COLUMN IF NOT EXISTS",
|
||||||
|
-- so running an ALTER for a column that already exists throws a harmless
|
||||||
|
-- "duplicate column name" error and changes nothing -- just run the flagged
|
||||||
|
-- subset. These are the 8 additive migration columns and nothing else; the
|
||||||
|
-- likes high-water-mark reset is intentionally NOT included.
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- ALTER TABLE Posts ADD COLUMN PostType TEXT;
|
||||||
|
-- ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;
|
||||||
|
-- ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;
|
||||||
|
-- ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;
|
||||||
|
-- ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;
|
||||||
|
-- ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;
|
||||||
|
-- ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;
|
||||||
|
-- ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';
|
||||||
Reference in New Issue
Block a user