From e18fb5ca100ffebaeebcc6fc340a275bed4e6389 Mon Sep 17 00:00:00 2001 From: jim Date: Fri, 2 Oct 2026 09:15:42 -0500 Subject: [PATCH] fix(posts): make HasImage raise-only to stop rescan churn TraverseDirectory's two AddPost call sites pass opposite hasImage values for one post (duplicate _N folders, /blog/private bodies on own-blog reblogs), and --ingest does the same across duplicate files. The overwrite flipped the flag 0->1->0 and bumped DateModified every pass. UpdatePost, AddPost's fallback and UpsertPostFromTextFile now only raise HasImage, matching ByLikes. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 26 +++++++++++++++++++------- URLNotesGrabberCORE/DataAccess.cs | 14 +++++++++----- URLNotesGrabberCORE/TL.db.md | 3 ++- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 04445fc..beee1d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -161,7 +161,7 @@ though the true content never changes. make the statement fire at all, and `DateModified` stays put - Deliberately narrow: only the literal `"."` is the sentinel. An explicit empty string from a real record still overwrites, same as before this fix. `postID`, `BlogName`, `hasImage`, - `ByLikes` are not part of this convention and are unaffected + `ByLikes` are not part of this convention; the two flags are raise-only instead (below) - If a new content field is added to `Posts`/`UpdatePost`, decide explicitly whether its source can legitimately supply `"."` as "field absent" before deciding whether it needs the same `CASE` treatment — don't assume every column needs it @@ -189,12 +189,24 @@ lands last for a given `(BlogName, PostID)` is arbitrary. sentinel. `G()` already distinguishes this from "present but blank" — a dictionary miss is `null`, an empty value after the prefix is `""` — so an explicitly blank field still overwrites -- `HasImage` is **not** guarded and remains a known gap: `IngestMode` always computes a - concrete `bool` (defaulting `false` when a file has no `Has Image:` line), so there is no - way for this function to tell "this format says no image" from "this format doesn't report - it at all" without changing the parameter to `bool?` and threading that through - `IngestMode`/`LegacyPostsDbImporter`. Fix this the same way if `--ingest` is observed - downgrading a post's `HasImage` from `1` to `0` +- `HasImage` cannot use the sentinel: `IngestMode` always computes a concrete `bool` + (defaulting `false` when a file has no `Has Image:` line), so this function cannot tell + "this format says no image" from "this format doesn't report it". It is raise-only instead + (below), which settles the same duplicate-file case without threading a `bool?` through + +**`Posts.HasImage` is raise-only (since 2026-10-02).** Every automated writer (`UpdatePost`, +`AddPost`'s fallback, `UpsertPostFromTextFile`) sets `HasImage = MAX(IFNULL(HasImage, 0), +@hasImage)` and guards the change-detection with `(@hasImage = 1 AND IFNULL(HasImage, 0) = 0)`, +the same shape `ByLikes` already used. No source sends a trustworthy "no image": a format +without a downloaded-files or `Has Image:` line reports `false` for a post that has one. + +- `TraverseDirectory`'s two `AddPost` call sites pass opposite values for one post — a + duplicate `_N` folder with and without a downloaded-files line, or a record with no + downloads whose body has `/blog/private` and whose `Reblog url` matches `ContainsList`. + Overwriting flipped the flag `0→1→0` on every no-argument scan, bumping `DateModified` + twice per pass and leaving whichever value ran last +- Lowering `HasImage` is a manual correction. `UpdatePostContentFields` (CorrectMode, restoring + from a `.bak`) still overwrites it, and that is deliberate ### Testing - No existing test suite; use xUnit if adding tests diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index ad4122a..d71aaa3 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -681,7 +681,7 @@ namespace URLNotesGrabberCORE { if (ownsConnection) connection.Open(); - string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage"; + string updateSql = "UPDATE Posts SET hasImage = 1, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND @hasImage = 1 AND IFNULL(hasImage, 0) = 0"; using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0); updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); @@ -1795,7 +1795,11 @@ namespace URLNotesGrabberCORE sql += "DateModified = @dateModified, "; sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, "; sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, "; - sql += "hasImage = @hasImage, "; + // Raise-only, like ByLikes. TraverseDirectory's two AddPost call sites pass + // opposite values for one post (a duplicate "_N" folder with and without a + // downloaded-files line, or a /blog/private body on an own-blog reblog), and + // overwriting flipped the flag -- and DateModified -- on every re-scan. + sql += "hasImage = MAX(IFNULL(hasImage, 0), @hasImage), "; // Fill in a missing type, never overwrite one. A type derived by --ingest from a // real export filename is authoritative; this path's type is only as good as the // folder it was crawled from, so it must not win over an existing value. @@ -1820,7 +1824,7 @@ namespace URLNotesGrabberCORE sql += "(@question <> '.' AND IFNULL(question, '') <> @question) OR "; sql += "(@answer <> '.' AND IFNULL(answer, '') <> @answer) OR "; sql += "(@title <> '.' AND IFNULL(title, '') <> @title) OR "; - sql += "IFNULL(hasImage, 0) <> @hasImage OR "; + sql += "(@hasImage = 1 AND IFNULL(hasImage, 0) = 0) OR "; sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR "; sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR "; sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL) OR "; @@ -2378,7 +2382,7 @@ namespace URLNotesGrabberCORE Answer = CASE WHEN @Answer IS NULL THEN Answer ELSE @Answer END, Title = CASE WHEN @Title IS NULL THEN Title ELSE @Title END, PostType = CASE WHEN @PostType IS NULL THEN PostType ELSE @PostType END, - HasImage = @HasImage, + HasImage = MAX(IFNULL(HasImage, 0), @HasImage), DateModified = @DateModified WHERE BlogName = @BlogName AND PostID = @PostID AND ( (@reblogURL IS NOT NULL AND IFNULL(reblogURL, '') <> @reblogURL) OR @@ -2400,7 +2404,7 @@ namespace URLNotesGrabberCORE (@Answer IS NOT NULL AND IFNULL(Answer, '') <> @Answer) OR (@Title IS NOT NULL AND IFNULL(Title, '') <> @Title) OR (@PostType IS NOT NULL AND IFNULL(PostType, '') <> @PostType) OR - IFNULL(HasImage, 0) <> @HasImage + (@HasImage = 1 AND IFNULL(HasImage, 0) = 0) )"; using (var cmd = new SQLiteCommand(updateSql, connection)) diff --git a/URLNotesGrabberCORE/TL.db.md b/URLNotesGrabberCORE/TL.db.md index ba695b4..2de5ad7 100644 --- a/URLNotesGrabberCORE/TL.db.md +++ b/URLNotesGrabberCORE/TL.db.md @@ -201,7 +201,8 @@ Notable: (`Fri, 14 Feb 2025 15:20:09 GMT`); every write path runs `PostDates.Normalize` to convert it, and `../normalize-postdate.sql` fixed the 4 rows written before that. - `HasImage = 1` on 14,026 rows. It records that the post *had* a picture, not that a - usable URL was kept, so it is not a reliable predictor that anything will render. + usable URL was kept, so it is not a reliable predictor that anything will render. It is + raise-only: no import or crawl lowers it, only a manual correction does. - `PhotoURL` is largely unused; in practice the image markup lives inside `Body`. - `NotFound = 1` on 4,712 rows — posts that have since been deleted upstream. - The content columns (`Body`, `Quote`, `Question`, `Answer`, …) are the heavy ones. List