diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index cc31e61..1924966 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -597,6 +597,7 @@ namespace URLNotesGrabberCORE { DBPath ??= GetDefaultDbPath(); postType = PostTypes.Normalize(postType); + postDate = PostDates.Normalize(postDate)!; try { AddBlog(blogName, byLikes, DBPath); } catch { } try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { } try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL, postType); } catch { } @@ -2249,6 +2250,7 @@ namespace URLNotesGrabberCORE // column. PostType is used as an output filename, so this is the invariant that keeps // a stray value from becoming a stray file. postType = PostTypes.Normalize(postType); + postDate = PostDates.Normalize(postDate); try { AddBlog(blogName, false, DBPath); } catch { } SQLiteConnection connection; @@ -2620,10 +2622,11 @@ namespace URLNotesGrabberCORE if (string.IsNullOrWhiteSpace(kvp.Value)) continue; string? column = MapPrefixToColumn(kvp.Key); if (column == null) continue; + string value = column == "PostDate" ? PostDates.Normalize(kvp.Value)! : kvp.Value; string paramName = "@p" + parameters.Count; setClauses.Add($"{column} = {paramName}"); changedClauses.Add($"IFNULL({column}, '') <> {paramName}"); - parameters.Add((paramName, kvp.Value)); + parameters.Add((paramName, value)); } if (setClauses.Count == 0) return false; diff --git a/URLNotesGrabberCORE/PostDates.cs b/URLNotesGrabberCORE/PostDates.cs new file mode 100644 index 0000000..5417bee --- /dev/null +++ b/URLNotesGrabberCORE/PostDates.cs @@ -0,0 +1,27 @@ +using System; +using System.Globalization; + +namespace URLNotesGrabberCORE +{ + /// + /// The single format for Posts.PostDate: "yyyy-MM-dd HH:mm:ss GMT", which is what the + /// Tumblr API sends and what nearly every row holds. Text-file exports can carry the + /// RFC 1123 form ("Fri, 14 Feb 2025 15:20:09 GMT") instead, which as text sorts on its + /// weekday name and falls outside every --fromDate/--toDate range comparison. + /// + /// Every path that writes PostDate routes through . Only the RFC 1123 + /// form is rewritten; anything else, including the "." no-change sentinel, passes through. + /// + public static class PostDates + { + public static string? Normalize(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return value; + string trimmed = value.Trim(); + if (DateTime.TryParseExact(trimmed, "r", CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out DateTime parsed)) + return parsed.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture) + " GMT"; + return value; + } + } +} diff --git a/URLNotesGrabberCORE/TL.db.md b/URLNotesGrabberCORE/TL.db.md index a4169cb..cc20372 100644 --- a/URLNotesGrabberCORE/TL.db.md +++ b/URLNotesGrabberCORE/TL.db.md @@ -154,6 +154,10 @@ Notable: on every row, and something has since started writing it. Anything that treated it as permanently unset, or derived the type from post content instead, should be re-examined against the live data. Rolodex still derives it. +- **`PostDate` is `yyyy-MM-dd HH:mm:ss GMT`** — UTC, as the Tumblr API sends it, unlike + the local-time `DateCreated`/`DateModified`. Text-file exports may carry RFC 1123 + (`Fri, 14 Feb 2025 15:20:09 GMT`); every write path runs `PostDates.Normalize` to + convert it, and `../normalize-postdate.sql` fixed the 4 rows written before that. - `HasImage = 1` on 14,026 rows. It records that the post *had* a picture, not that a usable URL was kept, so it is not a reliable predictor that anything will render. - `PhotoURL` is largely unused; in practice the image markup lives inside `Body`. diff --git a/normalize-postdate.sql b/normalize-postdate.sql new file mode 100644 index 0000000..c01b800 --- /dev/null +++ b/normalize-postdate.sql @@ -0,0 +1,31 @@ +-- normalize-postdate.sql +-- Rewrites Posts.PostDate values held in RFC 1123 form ("Fri, 14 Feb 2025 15:20:09 GMT") +-- into the column's canonical "yyyy-MM-dd HH:mm:ss GMT" (the Tumblr API's own format). +-- +-- As of 2026-09-21 this matched 4 rows, all zombaee, from one text-file import. As text +-- they sort on the weekday name and never satisfy --fromDate / --toDate comparisons. +-- New writes are normalized in code by PostDates.Normalize, so this is a one-off. +-- +-- Only PostDate changes. DateModified is left alone: the post content did not change. +-- +-- HOW TO RUN: back up TL.db, then from the URLNotesGrabberCORE project folder: +-- sqlite3 TL.db < ../normalize-postdate.sql + +SELECT 'before', COUNT(*) FROM Posts WHERE PostDate LIKE '___, __ ___ ____ __:__:__ GMT'; + +BEGIN; +UPDATE Posts + SET PostDate = substr(PostDate, 13, 4) || '-' || + printf('%02d', (instr('JanFebMarAprMayJunJulAugSepOctNovDec', substr(PostDate, 9, 3)) + 2) / 3) || '-' || + substr(PostDate, 6, 2) || ' ' || + substr(PostDate, 18) + WHERE PostDate LIKE '___, __ ___ ____ __:__:__ GMT' + AND instr('JanFebMarAprMayJunJulAugSepOctNovDec', substr(PostDate, 9, 3)) % 3 = 1; +COMMIT; + +-- VERIFY: expect 0, then a single shape '9999-99-99 99:99:99 GMT' (plus any NULL/blank) +SELECT 'after', COUNT(*) FROM Posts WHERE PostDate LIKE '___, %'; +SELECT CASE WHEN PostDate GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] GMT' + THEN 'yyyy-MM-dd HH:mm:ss GMT' ELSE IFNULL(PostDate, '(null)') END AS shape, + COUNT(*) + FROM Posts GROUP BY 1 ORDER BY 2 DESC;