From e1d2eb48c2870f73612974e0e4eab3ddb67d09cf Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 3 Aug 2026 08:53:10 -0500 Subject: [PATCH] fix: only bump DateModified when a value actually changed Seven UPDATE statements wrote DateModified unconditionally, so re-crawling or re-ingesting identical content marked Blogs, Posts and Notes rows as modified. Each now carries a WHERE guard covering every column in its SET list, so SQLite matches zero rows on a no-op. Guarded: AddPost's insert-failure fallback and blog stamp, AddNote's blog stamp, UpdateBlogLikesNewestTimestamp, UpdateNoteReplyText, UpsertPostFromTextFile, SetBlogTTFolderPath, UpdatePostContentFields. Also: - Blogs.DateAdded is no longer rewritten when a new post arrives for a known blog. A new post is not a new blog, and rewriting the column both destroyed the registration date and made every insert look like a change. - Posts.NotesGatheredDateTime is crawl bookkeeping that moves on every pass, so it no longer moves DateModified on its own. It is still written each pass, but the timestamp is wrapped in a CASE on the pre-UPDATE HasNotesGathered value so only the flag flipping counts. These statements now return 0 rows for "found but unchanged" as well as "not found"; CorrectMode's postsUpdated tally consequently counts rows actually changed, matching what its dry-run diff reports. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 34 +++++++++++++++++++ URLNotesGrabberCORE/DataAccess.cs | 55 ++++++++++++++++++++++++------- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3143484..33ddd5a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,6 +67,40 @@ say nothing about the item being fetched, so they must not be recorded as per-it - Do not add these columns from this app, and do not add them to the missing-column list in `verify-db-schema.sql` +### `DateModified` Tracks Real Changes Only +`Blogs.DateModified`, `Posts.DateModified` and `Notes.DateModified` must move only when a +column beside `DateModified` itself actually changed. Re-crawling or re-ingesting identical +content has to leave the row — and its timestamp — untouched, or downstream consumers cannot +tell a refreshed row from a rewritten one. + +- Enforce it in the `WHERE` clause, not in C#. Every `UPDATE` that sets `DateModified` ends + with an `AND ( <> @param OR ...)` term covering every column in its `SET` list, so + SQLite matches zero rows on a no-op and never writes +- Compare NULL-safely: `IFNULL(col, '') <> IFNULL(@param, '')` for text, + `IFNULL(col, 0) <> @param` for integer flags. A bare `col <> @param` is NULL on a NULL + column and silently skips the row that most needs writing +- Where NULL is not equivalent to the default, spell it out. The `HasBeenOutput = 0` stamps + use `(HasBeenOutput IS NULL OR HasBeenOutput <> 0)` because the selection queries test + `HasBeenOutput = 0`, which a NULL would never match +- Dynamic `SET` lists (`UpdatePostContentFields`) build the guard alongside the assignments + so the two lists cannot drift apart +- These statements now return 0 rows for "found but unchanged" as well as "not found". + Callers that read `ExecuteNonQuery()` must not treat 0 as "row missing" + +**`Posts.NotesGatheredDateTime` is crawl bookkeeping, not content.** It moves on every +`-collect` pass and says nothing about the post, so it must never move `DateModified` on its +own. `UpdatePostMarkNotesCollected` still writes it every pass but wraps the timestamp in +`DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE +DateModified END` — SQLite evaluates `SET` expressions against the pre-`UPDATE` row, so only +the flag flipping counts as a modification. Use this shape for any column that has to be +refreshed unconditionally without being a change. `Blogs.LikesLastRefreshed` is the +deliberate exception: a refresh pass is treated as a real event on the blog row. + +**`Blogs.DateAdded` is write-once.** `AddBlog`'s `INSERT` is the only place that sets it. A +new post arriving for a known blog reopens `HasBeenOutput` but must leave `DateAdded` alone — +a new post is not a new blog, and rewriting the column both destroys the registration date +and makes every insert look like a change. + ### Testing - No existing test suite; use xUnit if adding tests - Test critical logic: `ApiKeyPool` init, color parsing, config persistence diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index c56e0d4..f3a5bda 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -594,7 +594,7 @@ namespace URLNotesGrabberCORE { if (ownsConnection) connection.Open(); - string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID"; + string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage"; using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0); updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); @@ -619,16 +619,17 @@ namespace URLNotesGrabberCORE } } - // Only update HasBeenOutput and DateAdded if a new post was inserted + // Only reopen the blog for output if a new post was inserted. DateAdded records + // when the blog first entered the registry and is never rewritten here -- a new + // post is not a new blog. if (rowsInserted == 1) { try { - string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded, DateModified = @DateModified WHERE BlogName = @BlogName"; + string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)"; using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection)) { updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName); - updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); updateBlogCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); updateBlogCommand.ExecuteNonQuery(); } @@ -714,7 +715,9 @@ namespace URLNotesGrabberCORE { try { - string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName"; + // HasBeenOutput IS NULL still counts as a change: the selection queries + // test HasBeenOutput = 0, which a NULL would never match. + string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)"; using (var updateCommand = new SQLiteCommand(updateSql, connection2)) { updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName); @@ -1324,7 +1327,10 @@ namespace URLNotesGrabberCORE connection.Open(); //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID"; - string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)"; + // NotesGatheredDateTime is crawl bookkeeping -- it moves on every pass and says + // nothing about the post itself, so only the HasNotesGathered flag flipping + // counts as a modification. The CASE reads the pre-UPDATE value of the flag. + string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE DateModified END WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds()); @@ -1681,7 +1687,8 @@ namespace URLNotesGrabberCORE string sql = @"UPDATE Blogs SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest), DateModified = @modified - WHERE BlogName = @name"; + WHERE BlogName = @name + AND COALESCE(LikesNewestTimestamp, 0) < @newest"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@newest", newestTimestamp); @@ -1773,7 +1780,7 @@ namespace URLNotesGrabberCORE //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; // Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now. // Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text. - string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.')"; + string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.') AND (replyText IS NULL OR replyText <> @replyText)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@replyText", replyText ?? "?"); @@ -2030,7 +2037,28 @@ namespace URLNotesGrabberCORE PostType = @PostType, HasImage = @HasImage, DateModified = @DateModified - WHERE BlogName = @BlogName AND PostID = @PostID"; + WHERE BlogName = @BlogName AND PostID = @PostID AND ( + IFNULL(reblogURL, '') <> IFNULL(@reblogURL, '') OR + IFNULL(PostDate, '') <> IFNULL(@PostDate, '') OR + IFNULL(PostURL, '') <> IFNULL(@PostURL, '') OR + IFNULL(Slug, '') <> IFNULL(@Slug, '') OR + IFNULL(ReblogKey, '') <> IFNULL(@ReblogKey, '') OR + IFNULL(ReblogName, '') <> IFNULL(@ReblogName, '') OR + IFNULL(Summary, '') <> IFNULL(@Summary, '') OR + IFNULL(Quote, '') <> IFNULL(@Quote, '') OR + IFNULL(Body, '') <> IFNULL(@Body, '') OR + IFNULL(Tags, '') <> IFNULL(@Tags, '') OR + IFNULL(Link, '') <> IFNULL(@Link, '') OR + IFNULL(PhotoURL, '') <> IFNULL(@PhotoURL, '') OR + IFNULL(PhotoCaption, '') <> IFNULL(@PhotoCaption, '') OR + IFNULL(DownloadedFiles, '') <> IFNULL(@DownloadedFiles, '') OR + IFNULL(AudioCaption, '') <> IFNULL(@AudioCaption, '') OR + IFNULL(Question, '') <> IFNULL(@Question, '') OR + IFNULL(Answer, '') <> IFNULL(@Answer, '') OR + IFNULL(Title, '') <> IFNULL(@Title, '') OR + IFNULL(PostType, '') <> IFNULL(@PostType, '') OR + IFNULL(HasImage, 0) <> @HasImage + )"; using (var cmd = new SQLiteCommand(updateSql, connection)) { @@ -2222,7 +2250,7 @@ namespace URLNotesGrabberCORE using var connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); using var cmd = new SQLiteCommand( - "UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name", + "UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name AND IFNULL(TTFolderPath, '') <> IFNULL(@path, '')", connection); cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value); cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); @@ -2234,12 +2262,14 @@ namespace URLNotesGrabberCORE // ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty // values pulled from a BAK file. Only those columns + DateModified are written; // other content columns and all engagement columns are left intact. - // Returns true if a row was matched (and therefore updated). + // Returns true if a row was actually changed. A row whose columns already hold + // the incoming values is left alone, DateModified included. public static bool UpdatePostContentFields(string blogName, string postId, IDictionary fieldsToUpdate, string? DBPath = null) { DBPath ??= GetDefaultDbPath(); var setClauses = new List(); + var changedClauses = new List(); var parameters = new List<(string Name, object Value)>(); foreach (var kvp in fieldsToUpdate) @@ -2249,6 +2279,7 @@ namespace URLNotesGrabberCORE if (column == null) continue; string paramName = "@p" + parameters.Count; setClauses.Add($"{column} = {paramName}"); + changedClauses.Add($"IFNULL({column}, '') <> {paramName}"); parameters.Add((paramName, kvp.Value)); } @@ -2260,7 +2291,7 @@ namespace URLNotesGrabberCORE using var connection = new SQLiteConnection("Data Source=" + DBPath); connection.Open(); - string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID"; + string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID AND ({string.Join(" OR ", changedClauses)})"; using var cmd = new SQLiteCommand(sql, connection); foreach (var (name, value) in parameters) cmd.Parameters.AddWithValue(name, value);