fix(posts): make HasImage raise-only to stop rescan churn
TraverseDirectory's two AddPost call sites pass opposite hasImage values for one post (duplicate _N folders, /blog/private bodies on own-blog reblogs), and --ingest does the same across duplicate files. The overwrite flipped the flag 0->1->0 and bumped DateModified every pass. UpdatePost, AddPost's fallback and UpsertPostFromTextFile now only raise HasImage, matching ByLikes. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
5af284a49f
commit
e18fb5ca10
3 files changed
+30
-13
No files matched your search
@@ -681,7 +681,7 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
if (ownsConnection) connection.Open();
|
||||
|
||||
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage";
|
||||
string updateSql = "UPDATE Posts SET hasImage = 1, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND @hasImage = 1 AND IFNULL(hasImage, 0) = 0";
|
||||
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
||||
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
||||
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||
@@ -1795,7 +1795,11 @@ namespace URLNotesGrabberCORE
|
||||
sql += "DateModified = @dateModified, ";
|
||||
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
|
||||
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
|
||||
sql += "hasImage = @hasImage, ";
|
||||
// Raise-only, like ByLikes. TraverseDirectory's two AddPost call sites pass
|
||||
// opposite values for one post (a duplicate "_N" folder with and without a
|
||||
// downloaded-files line, or a /blog/private body on an own-blog reblog), and
|
||||
// overwriting flipped the flag -- and DateModified -- on every re-scan.
|
||||
sql += "hasImage = MAX(IFNULL(hasImage, 0), @hasImage), ";
|
||||
// Fill in a missing type, never overwrite one. A type derived by --ingest from a
|
||||
// real export filename is authoritative; this path's type is only as good as the
|
||||
// folder it was crawled from, so it must not win over an existing value.
|
||||
@@ -1820,7 +1824,7 @@ namespace URLNotesGrabberCORE
|
||||
sql += "(@question <> '.' AND IFNULL(question, '') <> @question) OR ";
|
||||
sql += "(@answer <> '.' AND IFNULL(answer, '') <> @answer) OR ";
|
||||
sql += "(@title <> '.' AND IFNULL(title, '') <> @title) OR ";
|
||||
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
|
||||
sql += "(@hasImage = 1 AND IFNULL(hasImage, 0) = 0) OR ";
|
||||
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
|
||||
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
|
||||
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL) OR ";
|
||||
@@ -2378,7 +2382,7 @@ namespace URLNotesGrabberCORE
|
||||
Answer = CASE WHEN @Answer IS NULL THEN Answer ELSE @Answer END,
|
||||
Title = CASE WHEN @Title IS NULL THEN Title ELSE @Title END,
|
||||
PostType = CASE WHEN @PostType IS NULL THEN PostType ELSE @PostType END,
|
||||
HasImage = @HasImage,
|
||||
HasImage = MAX(IFNULL(HasImage, 0), @HasImage),
|
||||
DateModified = @DateModified
|
||||
WHERE BlogName = @BlogName AND PostID = @PostID AND (
|
||||
(@reblogURL IS NOT NULL AND IFNULL(reblogURL, '') <> @reblogURL) OR
|
||||
@@ -2400,7 +2404,7 @@ namespace URLNotesGrabberCORE
|
||||
(@Answer IS NOT NULL AND IFNULL(Answer, '') <> @Answer) OR
|
||||
(@Title IS NOT NULL AND IFNULL(Title, '') <> @Title) OR
|
||||
(@PostType IS NOT NULL AND IFNULL(PostType, '') <> @PostType) OR
|
||||
IFNULL(HasImage, 0) <> @HasImage
|
||||
(@HasImage = 1 AND IFNULL(HasImage, 0) = 0)
|
||||
)";
|
||||
|
||||
using (var cmd = new SQLiteCommand(updateSql, connection))
|
||||
|
||||
@@ -201,7 +201,8 @@ Notable:
|
||||
(`Fri, 14 Feb 2025 15:20:09 GMT`); every write path runs `PostDates.Normalize` to
|
||||
convert it, and `../normalize-postdate.sql` fixed the 4 rows written before that.
|
||||
- `HasImage = 1` on 14,026 rows. It records that the post *had* a picture, not that a
|
||||
usable URL was kept, so it is not a reliable predictor that anything will render.
|
||||
usable URL was kept, so it is not a reliable predictor that anything will render. It is
|
||||
raise-only: no import or crawl lowers it, only a manual correction does.
|
||||
- `PhotoURL` is largely unused; in practice the image markup lives inside `Body`.
|
||||
- `NotFound = 1` on 4,712 rows — posts that have since been deleted upstream.
|
||||
- The content columns (`Body`, `Quote`, `Question`, `Answer`, …) are the heavy ones. List
|
||||
|
||||
Reference in new issue
Block a user