2 Commits
Author SHA1 Message Date
jim 49e581c136 Merge branch 'claude/post-modifieddate-rescan-f25914' into master 2026-10-02 09:15:53 -05:00
jimandClaude Opus 5.5 e18fb5ca10 fix(posts): make HasImage raise-only to stop rescan churn
TraverseDirectory's two AddPost call sites pass opposite hasImage values
for one post (duplicate _N folders, /blog/private bodies on own-blog
reblogs), and --ingest does the same across duplicate files. The
overwrite flipped the flag 0->1->0 and bumped DateModified every pass.
UpdatePost, AddPost's fallback and UpsertPostFromTextFile now only raise
HasImage, matching ByLikes.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-02 09:15:42 -05:00
3 changed files with 30 additions and 13 deletions

No files matched your search

+19 -7
View File
@@ -161,7 +161,7 @@ though the true content never changes.
make the statement fire at all, and `DateModified` stays put
- Deliberately narrow: only the literal `"."` is the sentinel. An explicit empty string from
a real record still overwrites, same as before this fix. `postID`, `BlogName`, `hasImage`,
`ByLikes` are not part of this convention and are unaffected
`ByLikes` are not part of this convention; the two flags are raise-only instead (below)
- If a new content field is added to `Posts`/`UpdatePost`, decide explicitly whether its
source can legitimately supply `"."` as "field absent" before deciding whether it needs
the same `CASE` treatment — don't assume every column needs it
@@ -189,12 +189,24 @@ lands last for a given `(BlogName, PostID)` is arbitrary.
sentinel. `G()` already distinguishes this from "present but blank" — a dictionary miss is
`null`, an empty value after the prefix is `""` — so an explicitly blank field still
overwrites
- `HasImage` is **not** guarded and remains a known gap: `IngestMode` always computes a
concrete `bool` (defaulting `false` when a file has no `Has Image:` line), so there is no
way for this function to tell "this format says no image" from "this format doesn't report
it at all" without changing the parameter to `bool?` and threading that through
`IngestMode`/`LegacyPostsDbImporter`. Fix this the same way if `--ingest` is observed
downgrading a post's `HasImage` from `1` to `0`
- `HasImage` cannot use the sentinel: `IngestMode` always computes a concrete `bool`
(defaulting `false` when a file has no `Has Image:` line), so this function cannot tell
"this format says no image" from "this format doesn't report it". It is raise-only instead
(below), which settles the same duplicate-file case without threading a `bool?` through
**`Posts.HasImage` is raise-only (since 2026-10-02).** Every automated writer (`UpdatePost`,
`AddPost`'s fallback, `UpsertPostFromTextFile`) sets `HasImage = MAX(IFNULL(HasImage, 0),
@hasImage)` and guards the change-detection with `(@hasImage = 1 AND IFNULL(HasImage, 0) = 0)`,
the same shape `ByLikes` already used. No source sends a trustworthy "no image": a format
without a downloaded-files or `Has Image:` line reports `false` for a post that has one.
- `TraverseDirectory`'s two `AddPost` call sites pass opposite values for one post — a
duplicate `_N` folder with and without a downloaded-files line, or a record with no
downloads whose body has `/blog/private` and whose `Reblog url` matches `ContainsList`.
Overwriting flipped the flag `0→1→0` on every no-argument scan, bumping `DateModified`
twice per pass and leaving whichever value ran last
- Lowering `HasImage` is a manual correction. `UpdatePostContentFields` (CorrectMode, restoring
from a `.bak`) still overwrites it, and that is deliberate
### Testing
- No existing test suite; use xUnit if adding tests
+9 -5
View File
@@ -681,7 +681,7 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage";
string updateSql = "UPDATE Posts SET hasImage = 1, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND @hasImage = 1 AND IFNULL(hasImage, 0) = 0";
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
@@ -1795,7 +1795,11 @@ namespace URLNotesGrabberCORE
sql += "DateModified = @dateModified, ";
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
sql += "hasImage = @hasImage, ";
// Raise-only, like ByLikes. TraverseDirectory's two AddPost call sites pass
// opposite values for one post (a duplicate "_N" folder with and without a
// downloaded-files line, or a /blog/private body on an own-blog reblog), and
// overwriting flipped the flag -- and DateModified -- on every re-scan.
sql += "hasImage = MAX(IFNULL(hasImage, 0), @hasImage), ";
// Fill in a missing type, never overwrite one. A type derived by --ingest from a
// real export filename is authoritative; this path's type is only as good as the
// folder it was crawled from, so it must not win over an existing value.
@@ -1820,7 +1824,7 @@ namespace URLNotesGrabberCORE
sql += "(@question <> '.' AND IFNULL(question, '') <> @question) OR ";
sql += "(@answer <> '.' AND IFNULL(answer, '') <> @answer) OR ";
sql += "(@title <> '.' AND IFNULL(title, '') <> @title) OR ";
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
sql += "(@hasImage = 1 AND IFNULL(hasImage, 0) = 0) OR ";
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL) OR ";
@@ -2378,7 +2382,7 @@ namespace URLNotesGrabberCORE
Answer = CASE WHEN @Answer IS NULL THEN Answer ELSE @Answer END,
Title = CASE WHEN @Title IS NULL THEN Title ELSE @Title END,
PostType = CASE WHEN @PostType IS NULL THEN PostType ELSE @PostType END,
HasImage = @HasImage,
HasImage = MAX(IFNULL(HasImage, 0), @HasImage),
DateModified = @DateModified
WHERE BlogName = @BlogName AND PostID = @PostID AND (
(@reblogURL IS NOT NULL AND IFNULL(reblogURL, '') <> @reblogURL) OR
@@ -2400,7 +2404,7 @@ namespace URLNotesGrabberCORE
(@Answer IS NOT NULL AND IFNULL(Answer, '') <> @Answer) OR
(@Title IS NOT NULL AND IFNULL(Title, '') <> @Title) OR
(@PostType IS NOT NULL AND IFNULL(PostType, '') <> @PostType) OR
IFNULL(HasImage, 0) <> @HasImage
(@HasImage = 1 AND IFNULL(HasImage, 0) = 0)
)";
using (var cmd = new SQLiteCommand(updateSql, connection))
+2 -1
View File
@@ -201,7 +201,8 @@ Notable:
(`Fri, 14 Feb 2025 15:20:09 GMT`); every write path runs `PostDates.Normalize` to
convert it, and `../normalize-postdate.sql` fixed the 4 rows written before that.
- `HasImage = 1` on 14,026 rows. It records that the post *had* a picture, not that a
usable URL was kept, so it is not a reliable predictor that anything will render.
usable URL was kept, so it is not a reliable predictor that anything will render. It is
raise-only: no import or crawl lowers it, only a manual correction does.
- `PhotoURL` is largely unused; in practice the image markup lives inside `Body`.
- `NotFound = 1` on 4,712 rows — posts that have since been deleted upstream.
- The content columns (`Body`, `Quote`, `Question`, `Answer`, …) are the heavy ones. List