jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(posttype): stop Unknown.txt and type posts at their source

Browse Source 
PostType becomes an output filename, so an unset or unvalidated value does
not stay a data problem -- it creates a file. OutputMode wrote untyped rows
to `PostType ?? "Unknown"`, IngestMode read that file back and derived the
literal type "Unknown" from its name, and the two would have regenerated
each other indefinitely.

Nothing was setting the type in the first place. AddPost -- the path every
notes/likes harvest goes through -- omitted PostType from its INSERT column
list entirely, so 1790 rows across 469 blogs had none. Ingest could never
repair them: it types a post only when it meets it inside a real export
file, and these posts appear in none.

Type at the source, from what each path actually knows:

- TraverseDirectory takes it from the filename it is already reading
  ("texts.txt" -> "texts"), the same rule ingest uses.
- CollectLikes has no file, so it reads the legacy-format `type` field that
  GrabLikes already requests with npf=false, mapped singular -> plural.
- AddPost/UpdatePost gained the plumbing to carry it. UpdatePost fills a
  missing type but never overwrites one, and its change-detection clause
  had to learn about PostType or the SET would be unreachable for a row
  whose content was already current.

PostTypes is the single source of truth: eight canonical names, and
anything else normalizes to null. Null is safe -- OutputMode skips those
rows -- while a stray value would have become a stray file. IngestMode and
TraverseDirectory now skip non-export .txt files outright, and the legacy
importer no longer passes a pre-column NULL straight back in.

For the rows already stored untyped, content is the only signal left, so
the migration infers from which columns they carry. Verified against a copy
of the live database: 1780 of 1790 typed, 10 left untyped for want of any
content at all, idempotent on a second pass. HasImage is deliberately not
consulted -- it is set on 12,420 of 19,828 known text posts.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
jim
2026-08-20 08:00:56 -05:00
co-authored by Claude Opus 5
parent 954ec353a5
commit 34da632e6a
6 changed files with 266 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+85 -6
View File
@@ -590,12 +590,16 @@ namespace URLNotesGrabberCORE
}
}
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
// postType: a canonical PostTypes name, or null when the caller has no trustworthy type.
// Null is stored as NULL rather than guessed at -- OutputMode skips untyped rows, so a
// null costs one export line, whereas a wrong value would create a wrongly named file.
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null, string? postType = null)
{
DBPath ??= GetDefaultDbPath();
postType = PostTypes.Normalize(postType);
try { AddBlog(blogName, byLikes, DBPath); } catch { }
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL); } catch { }
try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL, postType); } catch { }
SQLiteConnection connection;
bool ownsConnection;
@@ -637,9 +641,10 @@ namespace URLNotesGrabberCORE
RootBlogName,
RootURL,
HasImage,
ByLikes
ByLikes,
PostType
) VALUES (" +
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ", " + (postType == null ? "NULL" : Q(postType)) + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection);
int rowsInserted = 0;
@@ -1679,9 +1684,10 @@ namespace URLNotesGrabberCORE
return false;
}
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null, string? postType = null)
{
DBPath ??= GetDefaultDbPath();
postType = PostTypes.Normalize(postType);
SQLiteConnection connection;
bool ownsConnection;
@@ -1729,6 +1735,10 @@ namespace URLNotesGrabberCORE
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
sql += "hasImage = @hasImage, ";
// Fill in a missing type, never overwrite one. A type derived by --ingest from a
// real export filename is authoritative; this path's type is only as good as the
// folder it was crawled from, so it must not win over an existing value.
sql += "PostType = IFNULL(PostType, @postType), ";
sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) ";
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
sql += "(@postDate <> '.' AND IFNULL(postDate, '') <> @postDate) OR ";
@@ -1752,7 +1762,10 @@ namespace URLNotesGrabberCORE
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)";
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL) OR ";
// Without this the SET above is unreachable for a row whose content is already
// current: the UPDATE would not fire, and the type would stay NULL forever.
sql += "(PostType IS NULL AND @postType IS NOT NULL)";
sql += ")";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -1780,6 +1793,7 @@ namespace URLNotesGrabberCORE
command.Parameters.AddWithValue("@rootURL", string.IsNullOrWhiteSpace(rootURL) ? (object)DBNull.Value : rootURL);
command.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
command.Parameters.AddWithValue("@byLikes", byLikes ? 1 : 0);
command.Parameters.AddWithValue("@postType", (object?)postType ?? DBNull.Value);
command.Parameters.AddWithValue("@BlogName", blogName);
command.Parameters.AddWithValue("@PostID", postID);
@@ -2089,6 +2103,8 @@ namespace URLNotesGrabberCORE
cmd.ExecuteNonQuery();
Console.WriteLine("[Migration] Added PostType column to Posts table");
}
BackfillMissingPostTypes(connection);
}
catch (Exception ex)
{
@@ -2096,6 +2112,65 @@ namespace URLNotesGrabberCORE
}
}
/// <summary>
/// Types rows that carry no PostType, inferring it from which content columns they hold.
///
/// These are posts harvested from notes and likes rather than read out of a TumblThree
/// export, so no filename ever described them and --ingest can never reach them: it only
/// types a post it meets inside a real .txt. Content is the only signal they have.
///
/// Runs on every migration pass and is idempotent -- it only touches PostType IS NULL,
/// so a row typed once is never revisited. Rows whose columns give no signal at all stay
/// NULL and are skipped by OutputMode.
///
/// Mirrors PostTypes.InferFromContent; the two must agree. Notably HasImage is not
/// consulted, because most text posts carry it.
/// </summary>
private static void BackfillMissingPostTypes(SQLiteConnection connection)
{
const string set = @"
UPDATE Posts SET PostType = CASE
WHEN Has(Question) AND Has(Answer) THEN 'answers'
WHEN Has(Quote) THEN 'quotes'
WHEN Has(Link) THEN 'links'
WHEN Has(AudioCaption) THEN 'audios'
WHEN Has(Body) THEN 'texts'
WHEN Has(PhotoURL) OR Has(PhotoCaption) THEN 'images'
ELSE NULL END
WHERE PostType IS NULL";
// SQLite has no user-defined predicate here, so expand the "field supplied" test
// ("." is the not-supplied sentinel used throughout the export format) inline.
string sql = System.Text.RegularExpressions.Regex.Replace(
set, @"Has\((\w+)\)", "TRIM(IFNULL($1, '')) NOT IN ('', '.')");
try
{
long before;
using (var count = new SQLiteCommand("SELECT COUNT(*) FROM Posts WHERE PostType IS NULL", connection))
before = Convert.ToInt64(count.ExecuteScalar());
if (before == 0) return;
int changed;
using (var cmd = new SQLiteCommand(sql, connection))
changed = cmd.ExecuteNonQuery();
long after;
using (var count = new SQLiteCommand("SELECT COUNT(*) FROM Posts WHERE PostType IS NULL", connection))
after = Convert.ToInt64(count.ExecuteScalar());
if (changed > 0 || after != before)
Console.WriteLine($"[Migration] Backfilled PostType for {before - after} post(s); {after} still untyped (no content signal).");
}
catch (Exception ex)
{
// A failed backfill must not stop the run: untyped rows are skipped on export,
// which is inconvenient, not corrupting.
Console.WriteLine($"[Migration] PostType backfill failed: {ex.Message}");
}
}
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
// On collision, only content columns + PostType + DateModified are updated;
// engagement columns (ByLikes, RootBlogName, RootURL, HasNotesGathered, NotFound,
@@ -2126,6 +2201,10 @@ namespace URLNotesGrabberCORE
string? DBPath = null)
{
DBPath ??= GetDefaultDbPath();
// Central guarantee: whatever a caller believes, only a canonical type reaches the
// column. PostType is used as an output filename, so this is the invariant that keeps
// a stray value from becoming a stray file.
postType = PostTypes.Normalize(postType);
try { AddBlog(blogName, false, DBPath); } catch { }
SQLiteConnection connection;