Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef6629d86a | ||
|
|
6320e2c0c9 | ||
|
|
6136901cc7 | ||
|
|
83e35a2323 | ||
|
|
f0ccac6503 | ||
|
|
e1d2eb48c2 |
@@ -67,6 +67,93 @@ say nothing about the item being fetched, so they must not be recorded as per-it
|
|||||||
- Do not add these columns from this app, and do not add them to the missing-column list in
|
- Do not add these columns from this app, and do not add them to the missing-column list in
|
||||||
`verify-db-schema.sql`
|
`verify-db-schema.sql`
|
||||||
|
|
||||||
|
### `DateModified` Tracks Real Changes Only
|
||||||
|
`Blogs.DateModified`, `Posts.DateModified` and `Notes.DateModified` must move only when a
|
||||||
|
column beside `DateModified` itself actually changed. Re-crawling or re-ingesting identical
|
||||||
|
content has to leave the row — and its timestamp — untouched, or downstream consumers cannot
|
||||||
|
tell a refreshed row from a rewritten one.
|
||||||
|
|
||||||
|
- Enforce it in the `WHERE` clause, not in C#. Every `UPDATE` that sets `DateModified` ends
|
||||||
|
with an `AND (<col> <> @param OR ...)` term covering every column in its `SET` list, so
|
||||||
|
SQLite matches zero rows on a no-op and never writes
|
||||||
|
- Compare NULL-safely: `IFNULL(col, '') <> IFNULL(@param, '')` for text,
|
||||||
|
`IFNULL(col, 0) <> @param` for integer flags. A bare `col <> @param` is NULL on a NULL
|
||||||
|
column and silently skips the row that most needs writing
|
||||||
|
- Where NULL is not equivalent to the default, spell it out. The `HasBeenOutput = 0` stamps
|
||||||
|
use `(HasBeenOutput IS NULL OR HasBeenOutput <> 0)` because the selection queries test
|
||||||
|
`HasBeenOutput = 0`, which a NULL would never match
|
||||||
|
- Dynamic `SET` lists (`UpdatePostContentFields`) build the guard alongside the assignments
|
||||||
|
so the two lists cannot drift apart
|
||||||
|
- These statements now return 0 rows for "found but unchanged" as well as "not found".
|
||||||
|
Callers that read `ExecuteNonQuery()` must not treat 0 as "row missing"
|
||||||
|
|
||||||
|
**`Posts.NotesGatheredDateTime` is crawl bookkeeping, not content.** It moves on every
|
||||||
|
`-collect` pass and says nothing about the post, so it must never move `DateModified` on its
|
||||||
|
own. `UpdatePostMarkNotesCollected` still writes it every pass but wraps the timestamp in
|
||||||
|
`DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE
|
||||||
|
DateModified END` — SQLite evaluates `SET` expressions against the pre-`UPDATE` row, so only
|
||||||
|
the flag flipping counts as a modification. Use this shape for any column that has to be
|
||||||
|
refreshed unconditionally without being a change. `Blogs.LikesLastRefreshed` is the
|
||||||
|
deliberate exception: a refresh pass is treated as a real event on the blog row.
|
||||||
|
|
||||||
|
**`Blogs.DateAdded` is write-once.** `AddBlog`'s `INSERT` is the only place that sets it. A
|
||||||
|
new post arriving for a known blog reopens `HasBeenOutput` but must leave `DateAdded` alone —
|
||||||
|
a new post is not a new blog, and rewriting the column both destroys the registration date
|
||||||
|
and makes every insert look like a change.
|
||||||
|
|
||||||
|
**`"."` in a `Posts` content field means "not supplied", not "empty".** `ReblogRecord`
|
||||||
|
(`TraverseDirectory`'s parser for the local `.txt` export tree) and the `--likes` API path
|
||||||
|
both default every content field to the literal string `"."` when their source has no value
|
||||||
|
for it, then pass that straight to `UpdatePost`. A blog with two export folders in different
|
||||||
|
field formats (a duplicate `_2` folder, or a Tumblr export whose field set changed over time)
|
||||||
|
sends one record with a real `Title`/`Tags`/`Slug` and another with those fields `"."`
|
||||||
|
because that format never had a line for them — and without a guard, re-importing both on
|
||||||
|
every run flips the row back and forth forever, bumping `DateModified` on every pass even
|
||||||
|
though the true content never changes.
|
||||||
|
|
||||||
|
- Every content column in `UpdatePost`'s `SET` list is guarded the same way `RootBlogName`/
|
||||||
|
`RootURL` already were: `col = CASE WHEN @col = '.' THEN col ELSE @col END`. A `"."`
|
||||||
|
parameter leaves the existing value alone instead of overwriting it
|
||||||
|
- The change-detection `WHERE` clause carries the same exception —
|
||||||
|
`(@col <> '.' AND IFNULL(col, '') <> @col) OR ...` — so a `"."`-only difference does not
|
||||||
|
make the statement fire at all, and `DateModified` stays put
|
||||||
|
- Deliberately narrow: only the literal `"."` is the sentinel. An explicit empty string from
|
||||||
|
a real record still overwrites, same as before this fix. `postID`, `BlogName`, `hasImage`,
|
||||||
|
`ByLikes` are not part of this convention and are unaffected
|
||||||
|
- If a new content field is added to `Posts`/`UpdatePost`, decide explicitly whether its
|
||||||
|
source can legitimately supply `"."` as "field absent" before deciding whether it needs
|
||||||
|
the same `CASE` treatment — don't assume every column needs it
|
||||||
|
|
||||||
|
**`--ingest` (`UpsertPostFromTextFile`) uses `NULL`, not `"."`, for the same "field absent"
|
||||||
|
convention, and reconciling exactly this kind of duplicate IS the feature's job.**
|
||||||
|
`IngestMode` strips a trailing `_N` from the folder name before it ever reaches
|
||||||
|
`UpsertPostFromTextFile`, so a duplicate export folder collapses onto the same `BlogName` on
|
||||||
|
purpose — the whole point is to merge multiple differently-formatted files for the same post
|
||||||
|
into one row. `IngestMode.G(key)` returns `null` (not `"."`) when a field's line is absent
|
||||||
|
from a given file, `LegacyPostsDbImporter` passes `null` straight from a `NULL` source column,
|
||||||
|
and files are walked in raw filesystem enumeration order — never sorted — so which file's call
|
||||||
|
lands last for a given `(BlogName, PostID)` is arbitrary.
|
||||||
|
|
||||||
|
- Before the fix, the `UPDATE` branch set every column unconditionally, so whichever file
|
||||||
|
processed last for a `PostID` would null out every field its own record didn't carry —
|
||||||
|
silently erasing real `Title`/`Slug`/`Tags`/… another file had, the opposite of what
|
||||||
|
`--ingest` exists to do. This is worse than the `"."` case above: that one only caused
|
||||||
|
churn (the two writes canceled out); this one loses data, and which posts lose which
|
||||||
|
fields depends on filesystem enumeration order
|
||||||
|
- Same shape of fix, `NULL` instead of `"."` as the sentinel: `col = CASE WHEN @col IS NULL
|
||||||
|
THEN col ELSE @col END` in the `SET` list, `(@col IS NOT NULL AND IFNULL(col, '') <> @col)
|
||||||
|
OR ...` in the change-detection
|
||||||
|
- Same narrow rule: only `NULL` (the field's line was never present in this file) is the
|
||||||
|
sentinel. `G()` already distinguishes this from "present but blank" — a dictionary miss is
|
||||||
|
`null`, an empty value after the prefix is `""` — so an explicitly blank field still
|
||||||
|
overwrites
|
||||||
|
- `HasImage` is **not** guarded and remains a known gap: `IngestMode` always computes a
|
||||||
|
concrete `bool` (defaulting `false` when a file has no `Has Image:` line), so there is no
|
||||||
|
way for this function to tell "this format says no image" from "this format doesn't report
|
||||||
|
it at all" without changing the parameter to `bool?` and threading that through
|
||||||
|
`IngestMode`/`LegacyPostsDbImporter`. Fix this the same way if `--ingest` is observed
|
||||||
|
downgrading a post's `HasImage` from `1` to `0`
|
||||||
|
|
||||||
### Testing
|
### Testing
|
||||||
- No existing test suite; use xUnit if adding tests
|
- No existing test suite; use xUnit if adding tests
|
||||||
- Test critical logic: `ApiKeyPool` init, color parsing, config persistence
|
- Test critical logic: `ApiKeyPool` init, color parsing, config persistence
|
||||||
|
|||||||
@@ -598,7 +598,7 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
if (ownsConnection) connection.Open();
|
if (ownsConnection) connection.Open();
|
||||||
|
|
||||||
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
|
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage";
|
||||||
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
||||||
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
||||||
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
@@ -623,16 +623,17 @@ namespace URLNotesGrabberCORE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only update HasBeenOutput and DateAdded if a new post was inserted
|
// Only reopen the blog for output if a new post was inserted. DateAdded records
|
||||||
|
// when the blog first entered the registry and is never rewritten here -- a new
|
||||||
|
// post is not a new blog.
|
||||||
if (rowsInserted == 1)
|
if (rowsInserted == 1)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded, DateModified = @DateModified WHERE BlogName = @BlogName";
|
string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)";
|
||||||
using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
|
using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
|
||||||
{
|
{
|
||||||
updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
|
updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
|
||||||
updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
||||||
updateBlogCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
updateBlogCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
updateBlogCommand.ExecuteNonQuery();
|
updateBlogCommand.ExecuteNonQuery();
|
||||||
}
|
}
|
||||||
@@ -737,7 +738,9 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName";
|
// HasBeenOutput IS NULL still counts as a change: the selection queries
|
||||||
|
// test HasBeenOutput = 0, which a NULL would never match.
|
||||||
|
string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)";
|
||||||
using (var updateCommand = new SQLiteCommand(updateSql, connection2))
|
using (var updateCommand = new SQLiteCommand(updateSql, connection2))
|
||||||
{
|
{
|
||||||
updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
|
updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
|
||||||
@@ -1356,7 +1359,10 @@ namespace URLNotesGrabberCORE
|
|||||||
connection.Open();
|
connection.Open();
|
||||||
|
|
||||||
//string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
|
//string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
|
||||||
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
|
// NotesGatheredDateTime is crawl bookkeeping -- it moves on every pass and says
|
||||||
|
// nothing about the post itself, so only the HasNotesGathered flag flipping
|
||||||
|
// counts as a modification. The CASE reads the pre-UPDATE value of the flag.
|
||||||
|
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE DateModified END WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
||||||
@@ -1558,49 +1564,60 @@ namespace URLNotesGrabberCORE
|
|||||||
{
|
{
|
||||||
if (ownsConnection) connection.Open();
|
if (ownsConnection) connection.Open();
|
||||||
|
|
||||||
|
// "." is TraverseDirectory/ReblogRecord's sentinel for "this field had no
|
||||||
|
// matching line in this particular export file" -- not an empty value. A blog
|
||||||
|
// with two export files in different formats (e.g. an "_2" duplicate folder, or
|
||||||
|
// a Tumblr export whose field set changed over time) sends one record with a real
|
||||||
|
// Title and another with Title = "." for the same PostID, and re-importing both
|
||||||
|
// on every run must not let the "not supplied" record blank out what the other
|
||||||
|
// one has. Every content field below is CASE-guarded the same way RootBlogName/
|
||||||
|
// RootURL already were, and the change-detection ignores "." too so a "."-only
|
||||||
|
// difference doesn't fire the UPDATE (and bump DateModified) on its own. Only "."
|
||||||
|
// is treated as the sentinel -- an explicit empty string from a real field still
|
||||||
|
// overwrites, same as before.
|
||||||
string sql = "UPDATE Posts SET ";
|
string sql = "UPDATE Posts SET ";
|
||||||
sql += "postDate = @postDate, ";
|
sql += "postDate = CASE WHEN @postDate = '.' THEN postDate ELSE @postDate END, ";
|
||||||
sql += "reblogURL = @reblogURL, ";
|
sql += "reblogURL = CASE WHEN @reblogURL = '.' THEN reblogURL ELSE @reblogURL END, ";
|
||||||
sql += "postURL = @postURL, ";
|
sql += "postURL = CASE WHEN @postURL = '.' THEN postURL ELSE @postURL END, ";
|
||||||
sql += "slug = @slug, ";
|
sql += "slug = CASE WHEN @slug = '.' THEN slug ELSE @slug END, ";
|
||||||
sql += "reblogKey = @reblogKey, ";
|
sql += "reblogKey = CASE WHEN @reblogKey = '.' THEN reblogKey ELSE @reblogKey END, ";
|
||||||
sql += "reblogName = @reblogName, ";
|
sql += "reblogName = CASE WHEN @reblogName = '.' THEN reblogName ELSE @reblogName END, ";
|
||||||
sql += "summary = @summary, ";
|
sql += "summary = CASE WHEN @summary = '.' THEN summary ELSE @summary END, ";
|
||||||
sql += "quote = @quote, ";
|
sql += "quote = CASE WHEN @quote = '.' THEN quote ELSE @quote END, ";
|
||||||
sql += "body = @body, ";
|
sql += "body = CASE WHEN @body = '.' THEN body ELSE @body END, ";
|
||||||
sql += "tags = @tags, ";
|
sql += "tags = CASE WHEN @tags = '.' THEN tags ELSE @tags END, ";
|
||||||
sql += "link = @link, ";
|
sql += "link = CASE WHEN @link = '.' THEN link ELSE @link END, ";
|
||||||
sql += "photoURL = @photoURL, ";
|
sql += "photoURL = CASE WHEN @photoURL = '.' THEN photoURL ELSE @photoURL END, ";
|
||||||
sql += "photoCaption = @photoCaption, ";
|
sql += "photoCaption = CASE WHEN @photoCaption = '.' THEN photoCaption ELSE @photoCaption END, ";
|
||||||
sql += "downloadedFiles = @downloadedFiles, ";
|
sql += "downloadedFiles = CASE WHEN @downloadedFiles = '.' THEN downloadedFiles ELSE @downloadedFiles END, ";
|
||||||
sql += "audioCaption = @audioCaption, ";
|
sql += "audioCaption = CASE WHEN @audioCaption = '.' THEN audioCaption ELSE @audioCaption END, ";
|
||||||
sql += "question = @question, ";
|
sql += "question = CASE WHEN @question = '.' THEN question ELSE @question END, ";
|
||||||
sql += "answer = @answer, ";
|
sql += "answer = CASE WHEN @answer = '.' THEN answer ELSE @answer END, ";
|
||||||
sql += "title = @title, ";
|
sql += "title = CASE WHEN @title = '.' THEN title ELSE @title END, ";
|
||||||
sql += "DateModified = @dateModified, ";
|
sql += "DateModified = @dateModified, ";
|
||||||
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
|
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
|
||||||
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
|
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
|
||||||
sql += "hasImage = @hasImage, ";
|
sql += "hasImage = @hasImage, ";
|
||||||
sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) ";
|
sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) ";
|
||||||
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
|
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
|
||||||
sql += "IFNULL(postDate, '') <> @postDate OR ";
|
sql += "(@postDate <> '.' AND IFNULL(postDate, '') <> @postDate) OR ";
|
||||||
sql += "IFNULL(reblogURL, '') <> @reblogURL OR ";
|
sql += "(@reblogURL <> '.' AND IFNULL(reblogURL, '') <> @reblogURL) OR ";
|
||||||
sql += "IFNULL(postURL, '') <> @postURL OR ";
|
sql += "(@postURL <> '.' AND IFNULL(postURL, '') <> @postURL) OR ";
|
||||||
sql += "IFNULL(slug, '') <> @slug OR ";
|
sql += "(@slug <> '.' AND IFNULL(slug, '') <> @slug) OR ";
|
||||||
sql += "IFNULL(reblogKey, '') <> @reblogKey OR ";
|
sql += "(@reblogKey <> '.' AND IFNULL(reblogKey, '') <> @reblogKey) OR ";
|
||||||
sql += "IFNULL(reblogName, '') <> @reblogName OR ";
|
sql += "(@reblogName <> '.' AND IFNULL(reblogName, '') <> @reblogName) OR ";
|
||||||
sql += "IFNULL(summary, '') <> @summary OR ";
|
sql += "(@summary <> '.' AND IFNULL(summary, '') <> @summary) OR ";
|
||||||
sql += "IFNULL(quote, '') <> @quote OR ";
|
sql += "(@quote <> '.' AND IFNULL(quote, '') <> @quote) OR ";
|
||||||
sql += "IFNULL(body, '') <> @body OR ";
|
sql += "(@body <> '.' AND IFNULL(body, '') <> @body) OR ";
|
||||||
sql += "IFNULL(tags, '') <> @tags OR ";
|
sql += "(@tags <> '.' AND IFNULL(tags, '') <> @tags) OR ";
|
||||||
sql += "IFNULL(link, '') <> @link OR ";
|
sql += "(@link <> '.' AND IFNULL(link, '') <> @link) OR ";
|
||||||
sql += "IFNULL(photoURL, '') <> @photoURL OR ";
|
sql += "(@photoURL <> '.' AND IFNULL(photoURL, '') <> @photoURL) OR ";
|
||||||
sql += "IFNULL(photoCaption, '') <> @photoCaption OR ";
|
sql += "(@photoCaption <> '.' AND IFNULL(photoCaption, '') <> @photoCaption) OR ";
|
||||||
sql += "IFNULL(downloadedFiles, '') <> @downloadedFiles OR ";
|
sql += "(@downloadedFiles <> '.' AND IFNULL(downloadedFiles, '') <> @downloadedFiles) OR ";
|
||||||
sql += "IFNULL(audioCaption, '') <> @audioCaption OR ";
|
sql += "(@audioCaption <> '.' AND IFNULL(audioCaption, '') <> @audioCaption) OR ";
|
||||||
sql += "IFNULL(question, '') <> @question OR ";
|
sql += "(@question <> '.' AND IFNULL(question, '') <> @question) OR ";
|
||||||
sql += "IFNULL(answer, '') <> @answer OR ";
|
sql += "(@answer <> '.' AND IFNULL(answer, '') <> @answer) OR ";
|
||||||
sql += "IFNULL(title, '') <> @title OR ";
|
sql += "(@title <> '.' AND IFNULL(title, '') <> @title) OR ";
|
||||||
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
|
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
|
||||||
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
|
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
|
||||||
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
|
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
|
||||||
@@ -1713,7 +1730,8 @@ namespace URLNotesGrabberCORE
|
|||||||
string sql = @"UPDATE Blogs
|
string sql = @"UPDATE Blogs
|
||||||
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
|
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
|
||||||
DateModified = @modified
|
DateModified = @modified
|
||||||
WHERE BlogName = @name";
|
WHERE BlogName = @name
|
||||||
|
AND COALESCE(LikesNewestTimestamp, 0) < @newest";
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
command.Parameters.AddWithValue("@newest", newestTimestamp);
|
command.Parameters.AddWithValue("@newest", newestTimestamp);
|
||||||
@@ -1809,7 +1827,7 @@ namespace URLNotesGrabberCORE
|
|||||||
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
|
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
|
||||||
// Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
|
// Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
|
||||||
// Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text.
|
// Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text.
|
||||||
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.')";
|
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE noteBlogName = @noteBlogName AND ABS(TimeStamp - @TimeStamp) <= 5 AND Type = 'reply' AND (replyText IS NULL OR replyText = '' OR replyText = '.') AND (replyText IS NULL OR replyText <> @replyText)";
|
||||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||||
{
|
{
|
||||||
command.Parameters.AddWithValue("@replyText", replyText ?? "?");
|
command.Parameters.AddWithValue("@replyText", replyText ?? "?");
|
||||||
@@ -2044,29 +2062,65 @@ namespace URLNotesGrabberCORE
|
|||||||
|
|
||||||
if (rowsInserted == 0)
|
if (rowsInserted == 0)
|
||||||
{
|
{
|
||||||
|
// NULL is this function's sentinel for "this file's record had no line for
|
||||||
|
// that field" (IngestMode's G(key) misses return null; LegacyPostsDbImporter
|
||||||
|
// passes null straight from a NULL source column) -- it does not mean "clear
|
||||||
|
// this field". --ingest's entire reason to exist is reconciling multiple
|
||||||
|
// export files for the same (BlogName, PostID) -- IngestMode normalizes a
|
||||||
|
// "_2"-suffixed duplicate folder onto the same blog name specifically so a
|
||||||
|
// second, differently-formatted file for a post it already has gets merged in.
|
||||||
|
// Files are walked in filesystem enumeration order, not sorted, so which
|
||||||
|
// file's UpsertPostFromTextFile call runs last for a given PostID is
|
||||||
|
// effectively arbitrary. An unconditional SET here would let whichever file
|
||||||
|
// processed last silently null out every column its own record didn't carry,
|
||||||
|
// erasing real content the other file had -- the opposite of "clean up". Each
|
||||||
|
// column is CASE-guarded to keep the existing value when this call's parameter
|
||||||
|
// is NULL, and the change-detection ignores a NULL-vs-real mismatch the same
|
||||||
|
// way, so a partial record converges into the row instead of overwriting it.
|
||||||
string updateSql = @"UPDATE Posts SET
|
string updateSql = @"UPDATE Posts SET
|
||||||
reblogURL = @reblogURL,
|
reblogURL = CASE WHEN @reblogURL IS NULL THEN reblogURL ELSE @reblogURL END,
|
||||||
PostDate = @PostDate,
|
PostDate = CASE WHEN @PostDate IS NULL THEN PostDate ELSE @PostDate END,
|
||||||
PostURL = @PostURL,
|
PostURL = CASE WHEN @PostURL IS NULL THEN PostURL ELSE @PostURL END,
|
||||||
Slug = @Slug,
|
Slug = CASE WHEN @Slug IS NULL THEN Slug ELSE @Slug END,
|
||||||
ReblogKey = @ReblogKey,
|
ReblogKey = CASE WHEN @ReblogKey IS NULL THEN ReblogKey ELSE @ReblogKey END,
|
||||||
ReblogName = @ReblogName,
|
ReblogName = CASE WHEN @ReblogName IS NULL THEN ReblogName ELSE @ReblogName END,
|
||||||
Summary = @Summary,
|
Summary = CASE WHEN @Summary IS NULL THEN Summary ELSE @Summary END,
|
||||||
Quote = @Quote,
|
Quote = CASE WHEN @Quote IS NULL THEN Quote ELSE @Quote END,
|
||||||
Body = @Body,
|
Body = CASE WHEN @Body IS NULL THEN Body ELSE @Body END,
|
||||||
Tags = @Tags,
|
Tags = CASE WHEN @Tags IS NULL THEN Tags ELSE @Tags END,
|
||||||
Link = @Link,
|
Link = CASE WHEN @Link IS NULL THEN Link ELSE @Link END,
|
||||||
PhotoURL = @PhotoURL,
|
PhotoURL = CASE WHEN @PhotoURL IS NULL THEN PhotoURL ELSE @PhotoURL END,
|
||||||
PhotoCaption = @PhotoCaption,
|
PhotoCaption = CASE WHEN @PhotoCaption IS NULL THEN PhotoCaption ELSE @PhotoCaption END,
|
||||||
DownloadedFiles = @DownloadedFiles,
|
DownloadedFiles = CASE WHEN @DownloadedFiles IS NULL THEN DownloadedFiles ELSE @DownloadedFiles END,
|
||||||
AudioCaption = @AudioCaption,
|
AudioCaption = CASE WHEN @AudioCaption IS NULL THEN AudioCaption ELSE @AudioCaption END,
|
||||||
Question = @Question,
|
Question = CASE WHEN @Question IS NULL THEN Question ELSE @Question END,
|
||||||
Answer = @Answer,
|
Answer = CASE WHEN @Answer IS NULL THEN Answer ELSE @Answer END,
|
||||||
Title = @Title,
|
Title = CASE WHEN @Title IS NULL THEN Title ELSE @Title END,
|
||||||
PostType = @PostType,
|
PostType = CASE WHEN @PostType IS NULL THEN PostType ELSE @PostType END,
|
||||||
HasImage = @HasImage,
|
HasImage = @HasImage,
|
||||||
DateModified = @DateModified
|
DateModified = @DateModified
|
||||||
WHERE BlogName = @BlogName AND PostID = @PostID";
|
WHERE BlogName = @BlogName AND PostID = @PostID AND (
|
||||||
|
(@reblogURL IS NOT NULL AND IFNULL(reblogURL, '') <> @reblogURL) OR
|
||||||
|
(@PostDate IS NOT NULL AND IFNULL(PostDate, '') <> @PostDate) OR
|
||||||
|
(@PostURL IS NOT NULL AND IFNULL(PostURL, '') <> @PostURL) OR
|
||||||
|
(@Slug IS NOT NULL AND IFNULL(Slug, '') <> @Slug) OR
|
||||||
|
(@ReblogKey IS NOT NULL AND IFNULL(ReblogKey, '') <> @ReblogKey) OR
|
||||||
|
(@ReblogName IS NOT NULL AND IFNULL(ReblogName, '') <> @ReblogName) OR
|
||||||
|
(@Summary IS NOT NULL AND IFNULL(Summary, '') <> @Summary) OR
|
||||||
|
(@Quote IS NOT NULL AND IFNULL(Quote, '') <> @Quote) OR
|
||||||
|
(@Body IS NOT NULL AND IFNULL(Body, '') <> @Body) OR
|
||||||
|
(@Tags IS NOT NULL AND IFNULL(Tags, '') <> @Tags) OR
|
||||||
|
(@Link IS NOT NULL AND IFNULL(Link, '') <> @Link) OR
|
||||||
|
(@PhotoURL IS NOT NULL AND IFNULL(PhotoURL, '') <> @PhotoURL) OR
|
||||||
|
(@PhotoCaption IS NOT NULL AND IFNULL(PhotoCaption, '') <> @PhotoCaption) OR
|
||||||
|
(@DownloadedFiles IS NOT NULL AND IFNULL(DownloadedFiles, '') <> @DownloadedFiles) OR
|
||||||
|
(@AudioCaption IS NOT NULL AND IFNULL(AudioCaption, '') <> @AudioCaption) OR
|
||||||
|
(@Question IS NOT NULL AND IFNULL(Question, '') <> @Question) OR
|
||||||
|
(@Answer IS NOT NULL AND IFNULL(Answer, '') <> @Answer) OR
|
||||||
|
(@Title IS NOT NULL AND IFNULL(Title, '') <> @Title) OR
|
||||||
|
(@PostType IS NOT NULL AND IFNULL(PostType, '') <> @PostType) OR
|
||||||
|
IFNULL(HasImage, 0) <> @HasImage
|
||||||
|
)";
|
||||||
|
|
||||||
using (var cmd = new SQLiteCommand(updateSql, connection))
|
using (var cmd = new SQLiteCommand(updateSql, connection))
|
||||||
{
|
{
|
||||||
@@ -2258,7 +2312,7 @@ namespace URLNotesGrabberCORE
|
|||||||
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
connection.Open();
|
connection.Open();
|
||||||
using var cmd = new SQLiteCommand(
|
using var cmd = new SQLiteCommand(
|
||||||
"UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name",
|
"UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name AND IFNULL(TTFolderPath, '') <> IFNULL(@path, '')",
|
||||||
connection);
|
connection);
|
||||||
cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
@@ -2270,12 +2324,14 @@ namespace URLNotesGrabberCORE
|
|||||||
// ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty
|
// ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty
|
||||||
// values pulled from a BAK file. Only those columns + DateModified are written;
|
// values pulled from a BAK file. Only those columns + DateModified are written;
|
||||||
// other content columns and all engagement columns are left intact.
|
// other content columns and all engagement columns are left intact.
|
||||||
// Returns true if a row was matched (and therefore updated).
|
// Returns true if a row was actually changed. A row whose columns already hold
|
||||||
|
// the incoming values is left alone, DateModified included.
|
||||||
public static bool UpdatePostContentFields(string blogName, string postId, IDictionary<string, string> fieldsToUpdate, string? DBPath = null)
|
public static bool UpdatePostContentFields(string blogName, string postId, IDictionary<string, string> fieldsToUpdate, string? DBPath = null)
|
||||||
{
|
{
|
||||||
DBPath ??= GetDefaultDbPath();
|
DBPath ??= GetDefaultDbPath();
|
||||||
|
|
||||||
var setClauses = new List<string>();
|
var setClauses = new List<string>();
|
||||||
|
var changedClauses = new List<string>();
|
||||||
var parameters = new List<(string Name, object Value)>();
|
var parameters = new List<(string Name, object Value)>();
|
||||||
|
|
||||||
foreach (var kvp in fieldsToUpdate)
|
foreach (var kvp in fieldsToUpdate)
|
||||||
@@ -2285,6 +2341,7 @@ namespace URLNotesGrabberCORE
|
|||||||
if (column == null) continue;
|
if (column == null) continue;
|
||||||
string paramName = "@p" + parameters.Count;
|
string paramName = "@p" + parameters.Count;
|
||||||
setClauses.Add($"{column} = {paramName}");
|
setClauses.Add($"{column} = {paramName}");
|
||||||
|
changedClauses.Add($"IFNULL({column}, '') <> {paramName}");
|
||||||
parameters.Add((paramName, kvp.Value));
|
parameters.Add((paramName, kvp.Value));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2296,7 +2353,7 @@ namespace URLNotesGrabberCORE
|
|||||||
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||||
connection.Open();
|
connection.Open();
|
||||||
|
|
||||||
string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID";
|
string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID AND ({string.Join(" OR ", changedClauses)})";
|
||||||
using var cmd = new SQLiteCommand(sql, connection);
|
using var cmd = new SQLiteCommand(sql, connection);
|
||||||
foreach (var (name, value) in parameters)
|
foreach (var (name, value) in parameters)
|
||||||
cmd.Parameters.AddWithValue(name, value);
|
cmd.Parameters.AddWithValue(name, value);
|
||||||
|
|||||||
Reference in New Issue
Block a user