fix(posts): make HasImage raise-only to stop rescan churn

TraverseDirectory's two AddPost call sites pass opposite hasImage values
for one post (duplicate _N folders, /blog/private bodies on own-blog
reblogs), and --ingest does the same across duplicate files. The
overwrite flipped the flag 0->1->0 and bumped DateModified every pass.
UpdatePost, AddPost's fallback and UpsertPostFromTextFile now only raise
HasImage, matching ByLikes.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jimandClaude Opus 5.5 committed 2026-10-02 09:15:42 -05:00
1 parent 5af284a49f
commit e18fb5ca10
3 files changed
+30 -13

No files matched your search

+9 -5
View File
@@ -681,7 +681,7 @@ namespace URLNotesGrabberCORE
{
if (ownsConnection) connection.Open();
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage";
string updateSql = "UPDATE Posts SET hasImage = 1, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND @hasImage = 1 AND IFNULL(hasImage, 0) = 0";
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
@@ -1795,7 +1795,11 @@ namespace URLNotesGrabberCORE
sql += "DateModified = @dateModified, ";
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
sql += "hasImage = @hasImage, ";
// Raise-only, like ByLikes. TraverseDirectory's two AddPost call sites pass
// opposite values for one post (a duplicate "_N" folder with and without a
// downloaded-files line, or a /blog/private body on an own-blog reblog), and
// overwriting flipped the flag -- and DateModified -- on every re-scan.
sql += "hasImage = MAX(IFNULL(hasImage, 0), @hasImage), ";
// Fill in a missing type, never overwrite one. A type derived by --ingest from a
// real export filename is authoritative; this path's type is only as good as the
// folder it was crawled from, so it must not win over an existing value.
@@ -1820,7 +1824,7 @@ namespace URLNotesGrabberCORE
sql += "(@question <> '.' AND IFNULL(question, '') <> @question) OR ";
sql += "(@answer <> '.' AND IFNULL(answer, '') <> @answer) OR ";
sql += "(@title <> '.' AND IFNULL(title, '') <> @title) OR ";
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
sql += "(@hasImage = 1 AND IFNULL(hasImage, 0) = 0) OR ";
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL) OR ";
@@ -2378,7 +2382,7 @@ namespace URLNotesGrabberCORE
Answer = CASE WHEN @Answer IS NULL THEN Answer ELSE @Answer END,
Title = CASE WHEN @Title IS NULL THEN Title ELSE @Title END,
PostType = CASE WHEN @PostType IS NULL THEN PostType ELSE @PostType END,
HasImage = @HasImage,
HasImage = MAX(IFNULL(HasImage, 0), @HasImage),
DateModified = @DateModified
WHERE BlogName = @BlogName AND PostID = @PostID AND (
(@reblogURL IS NOT NULL AND IFNULL(reblogURL, '') <> @reblogURL) OR
@@ -2400,7 +2404,7 @@ namespace URLNotesGrabberCORE
(@Answer IS NOT NULL AND IFNULL(Answer, '') <> @Answer) OR
(@Title IS NOT NULL AND IFNULL(Title, '') <> @Title) OR
(@PostType IS NOT NULL AND IFNULL(PostType, '') <> @PostType) OR
IFNULL(HasImage, 0) <> @HasImage
(@HasImage = 1 AND IFNULL(HasImage, 0) = 0)
)";
using (var cmd = new SQLiteCommand(updateSql, connection))
+2 -1
View File
@@ -201,7 +201,8 @@ Notable:
(`Fri, 14 Feb 2025 15:20:09 GMT`); every write path runs `PostDates.Normalize` to
convert it, and `../normalize-postdate.sql` fixed the 4 rows written before that.
- `HasImage = 1` on 14,026 rows. It records that the post *had* a picture, not that a
usable URL was kept, so it is not a reliable predictor that anything will render.
usable URL was kept, so it is not a reliable predictor that anything will render. It is
raise-only: no import or crawl lowers it, only a manual correction does.
- `PhotoURL` is largely unused; in practice the image markup lives inside `Body`.
- `NotFound = 1` on 4,712 rows — posts that have since been deleted upstream.
- The content columns (`Body`, `Quote`, `Question`, `Answer`, …) are the heavy ones. List