Merge GitTea/master into master

Seven UPDATE statements wrote DateModified unconditionally, so re-crawling
or re-ingesting identical content marked Blogs, Posts and Notes rows as
modified. Each now carries a WHERE guard covering every column in its SET
list, so SQLite matches zero rows on a no-op.

Guarded: AddPost's insert-failure fallback and blog stamp, AddNote's blog
stamp, UpdateBlogLikesNewestTimestamp, UpdateNoteReplyText,
UpsertPostFromTextFile, SetBlogTTFolderPath, UpdatePostContentFields.

Also:
- Blogs.DateAdded is no longer rewritten when a new post arrives for a
  known blog. A new post is not a new blog, and rewriting the column both
  destroyed the registration date and made every insert look like a change.
- Posts.NotesGatheredDateTime is crawl bookkeeping that moves on every
  pass, so it no longer moves DateModified on its own. It is still written
  each pass, but the timestamp is wrapped in a CASE on the pre-UPDATE
  HasNotesGathered value so only the flag flipping counts.

These statements now return 0 rows for "found but unchanged" as well as
"not found"; CorrectMode's postsUpdated tally consequently counts rows
actually changed, matching what its dry-run diff reports.

Co-Authored-By: Claude Opus 5 <[email protected]>

AGENTS.md

@@ -67,6 +67,40 @@ say nothing about the item being fetched, so they must not be recorded as per-it
 - Do not add these columns from this app, and do not add them to the missing-column list in
   `verify-db-schema.sql`
 
+### `DateModified` Tracks Real Changes Only
+`Blogs.DateModified`, `Posts.DateModified` and `Notes.DateModified` must move only when a
+column beside `DateModified` itself actually changed. Re-crawling or re-ingesting identical
+content has to leave the row — and its timestamp — untouched, or downstream consumers cannot
+tell a refreshed row from a rewritten one.
+
+- Enforce it in the `WHERE` clause, not in C#. Every `UPDATE` that sets `DateModified` ends
+  with an `AND (<col> <> @param OR ...)` term covering every column in its `SET` list, so
+  SQLite matches zero rows on a no-op and never writes
+- Compare NULL-safely: `IFNULL(col, '') <> IFNULL(@param, '')` for text,
+  `IFNULL(col, 0) <> @param` for integer flags. A bare `col <> @param` is NULL on a NULL
+  column and silently skips the row that most needs writing
+- Where NULL is not equivalent to the default, spell it out. The `HasBeenOutput = 0` stamps
+  use `(HasBeenOutput IS NULL OR HasBeenOutput <> 0)` because the selection queries test
+  `HasBeenOutput = 0`, which a NULL would never match
+- Dynamic `SET` lists (`UpdatePostContentFields`) build the guard alongside the assignments
+  so the two lists cannot drift apart
+- These statements now return 0 rows for "found but unchanged" as well as "not found".
+  Callers that read `ExecuteNonQuery()` must not treat 0 as "row missing"
+
+**`Posts.NotesGatheredDateTime` is crawl bookkeeping, not content.** It moves on every
+`-collect` pass and says nothing about the post, so it must never move `DateModified` on its
+own. `UpdatePostMarkNotesCollected` still writes it every pass but wraps the timestamp in
+`DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE
+DateModified END` — SQLite evaluates `SET` expressions against the pre-`UPDATE` row, so only
+the flag flipping counts as a modification. Use this shape for any column that has to be
+refreshed unconditionally without being a change. `Blogs.LikesLastRefreshed` is the
+deliberate exception: a refresh pass is treated as a real event on the blog row.
+
+**`Blogs.DateAdded` is write-once.** `AddBlog`'s `INSERT` is the only place that sets it. A
+new post arriving for a known blog reopens `HasBeenOutput` but must leave `DateAdded` alone —
+a new post is not a new blog, and rewriting the column both destroys the registration date
+and makes every insert look like a change.
+
 ### Testing
 - No existing test suite; use xUnit if adding tests
 - Test critical logic: `ApiKeyPool` init, color parsing, config persistence

URLNotesGrabberCORE/DataAccess.cs

@@ -598,7 +598,7 @@ namespace URLNotesGrabberCORE
                         {
                             if (ownsConnection) connection.Open();
 
-                            string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
+                            string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage";
                             using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
                             updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
                             updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
@@ -623,16 +623,17 @@ namespace URLNotesGrabberCORE
                     }
                 }
 
-                // Only update HasBeenOutput and DateAdded if a new post was inserted
+                // Only reopen the blog for output if a new post was inserted. DateAdded records
+                // when the blog first entered the registry and is never rewritten here -- a new
+                // post is not a new blog.
                 if (rowsInserted == 1)
                 {
                     try
                     {
-                        string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded, DateModified = @DateModified WHERE BlogName = @BlogName";
+                        string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)";
                         using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
                         {
                             updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
-                            updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                             updateBlogCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                             updateBlogCommand.ExecuteNonQuery();
                         }
@@ -737,7 +738,9 @@ namespace URLNotesGrabberCORE
                     {
                         try
                         {
-                            string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName";
+                            // HasBeenOutput IS NULL still counts as a change: the selection queries
+                            // test HasBeenOutput = 0, which a NULL would never match.
+                            string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)";
                             using (var updateCommand = new SQLiteCommand(updateSql, connection2))
                             {
                                 updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
@@ -1356,7 +1359,10 @@ namespace URLNotesGrabberCORE
                 connection.Open();
 
                 //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
-                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
+                // NotesGatheredDateTime is crawl bookkeeping -- it moves on every pass and says
+                // nothing about the post itself, so only the HasNotesGathered flag flipping
+                // counts as a modification. The CASE reads the pre-UPDATE value of the flag.
+                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE DateModified END WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
@@ -1713,7 +1719,8 @@ namespace URLNotesGrabberCORE
                 string sql = @"UPDATE Blogs
                                   SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
                                       DateModified = @modified
-                                WHERE BlogName = @name";
+                                WHERE BlogName = @name
+                                  AND COALESCE(LikesNewestTimestamp, 0) < @newest";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@newest", newestTimestamp);
@@ -1809,7 +1816,7 @@ namespace URLNotesGrabberCORE
                 //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
                 // Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
                 // Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text.
-                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.')";
+                string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.') AND (replyText IS NULL OR replyText <> @replyText)";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     command.Parameters.AddWithValue("@replyText", replyText ?? "?");
@@ -2066,7 +2073,28 @@ namespace URLNotesGrabberCORE
                                 PostType = @PostType,
                                 HasImage = @HasImage,
                                 DateModified = @DateModified
-                                WHERE BlogName = @BlogName AND PostID = @PostID";
+                                WHERE BlogName = @BlogName AND PostID = @PostID AND (
+                                    IFNULL(reblogURL, '') <> IFNULL(@reblogURL, '') OR
+                                    IFNULL(PostDate, '') <> IFNULL(@PostDate, '') OR
+                                    IFNULL(PostURL, '') <> IFNULL(@PostURL, '') OR
+                                    IFNULL(Slug, '') <> IFNULL(@Slug, '') OR
+                                    IFNULL(ReblogKey, '') <> IFNULL(@ReblogKey, '') OR
+                                    IFNULL(ReblogName, '') <> IFNULL(@ReblogName, '') OR
+                                    IFNULL(Summary, '') <> IFNULL(@Summary, '') OR
+                                    IFNULL(Quote, '') <> IFNULL(@Quote, '') OR
+                                    IFNULL(Body, '') <> IFNULL(@Body, '') OR
+                                    IFNULL(Tags, '') <> IFNULL(@Tags, '') OR
+                                    IFNULL(Link, '') <> IFNULL(@Link, '') OR
+                                    IFNULL(PhotoURL, '') <> IFNULL(@PhotoURL, '') OR
+                                    IFNULL(PhotoCaption, '') <> IFNULL(@PhotoCaption, '') OR
+                                    IFNULL(DownloadedFiles, '') <> IFNULL(@DownloadedFiles, '') OR
+                                    IFNULL(AudioCaption, '') <> IFNULL(@AudioCaption, '') OR
+                                    IFNULL(Question, '') <> IFNULL(@Question, '') OR
+                                    IFNULL(Answer, '') <> IFNULL(@Answer, '') OR
+                                    IFNULL(Title, '') <> IFNULL(@Title, '') OR
+                                    IFNULL(PostType, '') <> IFNULL(@PostType, '') OR
+                                    IFNULL(HasImage, 0) <> @HasImage
+                                )";
 
                     using (var cmd = new SQLiteCommand(updateSql, connection))
                     {
@@ -2258,7 +2286,7 @@ namespace URLNotesGrabberCORE
             using var connection = new SQLiteConnection("Data Source=" + DBPath);
             connection.Open();
             using var cmd = new SQLiteCommand(
-                "UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name",
+                "UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name AND IFNULL(TTFolderPath, '') <> IFNULL(@path, '')",
                 connection);
             cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
             cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
@@ -2270,12 +2298,14 @@ namespace URLNotesGrabberCORE
         // ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty
         // values pulled from a BAK file. Only those columns + DateModified are written;
         // other content columns and all engagement columns are left intact.
-        // Returns true if a row was matched (and therefore updated).
+        // Returns true if a row was actually changed. A row whose columns already hold
+        // the incoming values is left alone, DateModified included.
         public static bool UpdatePostContentFields(string blogName, string postId, IDictionary<string, string> fieldsToUpdate, string? DBPath = null)
         {
             DBPath ??= GetDefaultDbPath();
 
             var setClauses = new List<string>();
+            var changedClauses = new List<string>();
             var parameters = new List<(string Name, object Value)>();
 
             foreach (var kvp in fieldsToUpdate)
@@ -2285,6 +2315,7 @@ namespace URLNotesGrabberCORE
                 if (column == null) continue;
                 string paramName = "@p" + parameters.Count;
                 setClauses.Add($"{column} = {paramName}");
+                changedClauses.Add($"IFNULL({column}, '') <> {paramName}");
                 parameters.Add((paramName, kvp.Value));
             }
 
@@ -2296,7 +2327,7 @@ namespace URLNotesGrabberCORE
             using var connection = new SQLiteConnection("Data Source=" + DBPath);
             connection.Open();
 
-            string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID";
+            string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID AND ({string.Join(" OR ", changedClauses)})";
             using var cmd = new SQLiteCommand(sql, connection);
             foreach (var (name, value) in parameters)
                 cmd.Parameters.AddWithValue(name, value);