Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4d37999f8e | ||
|
|
387c023900 | ||
|
|
a9bd5a4c37 | ||
|
|
70b32dfc89 | ||
|
|
a2763d0026 |
Binary file not shown.
+39
-6
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="C:/Users/jim/Nextcloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="4305"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077200000379fc0100000006fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000000007720000011700fffffffb000000160064006f0063006b00420072006f00770073006500340100000000000005f40000000000000000000007720000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="ApiKeyPoolMeta" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="29"/><column index="2" value="64"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/><column index="5" value="51"/><column index="6" value="71"/><column index="7" value="85"/><column index="8" value="156"/><column index="9" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="28" mode="1"/></sort><column_widths><column index="1" value="241"/><column index="2" value="148"/><column index="3" value="126"/><column index="4" value="300"/><column index="5" value="75"/><column index="6" value="187"/><column index="7" value="159"/><column index="8" value="75"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="78"/><column index="12" value="249"/><column index="13" value="300"/><column index="14" value="53"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="41"/><column index="18" value="75"/><column index="19" value="96"/><column index="20" value="300"/><column index="21" value="96"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="42"/><column index="25" value="60"/><column index="26" value="218"/><column index="27" value="920"/><column index="28" value="156"/><column index="29" value="156"/></column_widths><filter_values><column index="24" value="=1"/><column index="28" value=">2026-05-06 20:00:01"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="4486"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd0000000100000002000005470000029afc0100000006fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000000005470000011100fffffffb000000160064006f0063006b00420072006f00770073006500340100000000000005f40000000000000000000005470000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="ApiKeyPoolMeta" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="29"/><column index="2" value="64"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="7" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="108"/><column index="3" value="63"/><column index="4" value="156"/><column index="5" value="60"/><column index="6" value="81"/><column index="7" value="85"/><column index="8" value="156"/><column index="9" value="156"/><column index="10" value="151"/><column index="11" value="125"/><column index="12" value="129"/></column_widths><filter_values><column index="4" value="1"/><column index="7" value=">2026-05-27 17:22:36"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort/><column_widths><column index="1" value="198"/><column index="2" value="144"/><column index="3" value="251"/><column index="4" value="84"/><column index="5" value="53"/><column index="6" value="300"/><column index="7" value="116"/><column index="8" value="152"/><column index="9" value="89"/><column index="10" value="63"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="14" mode="1"/></sort><column_widths><column index="1" value="236"/><column index="2" value="144"/><column index="3" value="126"/><column index="4" value="32"/><column index="5" value="32"/><column index="6" value="32"/><column index="7" value="32"/><column index="8" value="32"/><column index="9" value="0"/><column index="10" value="0"/><column index="11" value="0"/><column index="12" value="243"/><column index="13" value="300"/><column index="14" value="53"/><column index="15" value="37351"/><column index="16" value="300"/><column index="17" value="41"/><column index="18" value="75"/><column index="19" value="96"/><column index="20" value="300"/><column index="21" value="96"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="548"/><column index="25" value="60"/><column index="26" value="213"/><column index="27" value="532"/><column index="28" value="152"/><column index="29" value="152"/><column index="30" value="69"/><column index="31" value="63"/></column_widths><filter_values><column index="2" value="0"/><column index="1" value="734568371821084672"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns><column index="9" value="1"/><column index="10" value="1"/><column index="11" value="1"/></hidden_columns><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
|
||||
SET HasNotesGathered = 0
|
||||
WHERE (BlogName, PostID) IN (
|
||||
SELECT p.BlogName, p.PostID
|
||||
@@ -29,11 +29,12 @@ blogname in
|
||||
'nudenymph',
|
||||
'caylachief'
|
||||
|
||||
)</sql><sql name="New Notes">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
|
||||
from Notes
|
||||
)</sql><sql name="New Notes">select P.slug, N.replyText, n.RootBlogName, n.PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, n.RootBlogName || '.tumblr.com/post/' || n.postid, datetime(timestamp, 'unixepoch')
|
||||
from Notes N inner join Posts P on p.PostID = n.PostID
|
||||
where
|
||||
DatetimeCrawled > '2026-05-14 02:50:05' --and type like 'r%'
|
||||
order by DatetimeCrawled desc</sql><sql name="Pull Blogs*">SELECT distinct␍
|
||||
DatetimeCrawled > '2026-08-07 11:47:22' and type like 'r%'
|
||||
and P.IsActive = 1
|
||||
order by n.DatetimeCrawled</sql><sql name="Pull Blogs">SELECT distinct
|
||||
'''' || blogname || ''',',
|
||||
blogs.*
|
||||
, blogname || '.tumblr.com'
|
||||
@@ -64,4 +65,36 @@ JOIN ReplyCounts c ON n.NoteBlogName = c.NoteBlogName
|
||||
where replyText <> '.' and type <> 'reply'
|
||||
--AND N.NoteBlogName NOT IN ( 'roadblocker21', 'thesaddemon666', 'edwardabbeyhoffman', 'tattedsoldier20', 'zomb-eh', 'animalistic13', 'indken', 'maccloud1592',
|
||||
--'moss-wizard', 'supertrucker12682', 'exploringthrupics', 'padeyepete' )
|
||||
order by c.DistinctReplyCount desc, n.NoteBlogName, n.DateModified desc, replyText, RootBlogName, PostID</sql><current_tab id="3"/></tab_sql></sqlb_project>
|
||||
order by c.DistinctReplyCount desc, n.NoteBlogName, n.DateModified desc, replyText, RootBlogName, PostID</sql><sql name="Collect">WITH PostsWithCount AS ( SELECT P.BlogName, P.PostID, 1925013599 AS LatestNoteTimestamp, P.NotesGatheredDateTime, COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT, P.HasNotesGathered, P.NotFound, P.PostDate FROM Posts P WHERE COALESCE(P.IsActive, 1) = 1 ), Unioned AS ( SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE NotFound = 0 AND HasNotesGathered = 0 UNION SELECT BlogName, PostID, LatestNoteTimestamp, NotesGatheredDateTime, CNT, PostDate FROM PostsWithCount WHERE BlogName = 'zomb-eh' AND NotFound = 0 AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days') ) SELECT U.BlogName, U.PostID, U.LatestNoteTimestamp, U.NotesGatheredDateTime, U.CNT FROM Unioned U WHERE (U.NotesGatheredDateTime < 1786134037 OR U.NotesGatheredDateTime IS NULL) ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;</sql><sql name="Del Posts">delete from posts where postid in
|
||||
(
|
||||
'741662499571728384',
|
||||
178892849664,
|
||||
178264721139,
|
||||
177012868749,
|
||||
169950081964,
|
||||
755440787056099328
|
||||
)</sql><sql name="notes NO post*">select *
|
||||
-- delete
|
||||
from notes
|
||||
where postid not in (select distinct postid from posts where IsActive = 1)</sql><sql name="SQL 9">SELECT
|
||||
*
|
||||
FROM
|
||||
POSTS P
|
||||
WHERE
|
||||
P.ByLikes = 1
|
||||
AND
|
||||
P.DateCreated > '2026-05-26 17:47:32'
|
||||
ORDER BY
|
||||
P.DateCreated desc</sql><sql name="SQL 13">update posts set IsActive = 0 where blogname IN ( 'shoebiedoo', 'redheaded-girlygirl', 'xlittle-ghost' )</sql><sql name="SQL 14*">update Posts␍
|
||||
set IsActive = 0␍
|
||||
where postid in␍
|
||||
(␍
|
||||
␍
|
||||
␍
|
||||
'731937314675310592'␍
|
||||
␍
|
||||
␍
|
||||
␍
|
||||
)␍
|
||||
␍
|
||||
</sql><current_tab id="7"/></tab_sql></sqlb_project>
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -121,22 +121,56 @@ Notable:
|
||||
|
||||
```sql
|
||||
CREATE TABLE "Notes" (
|
||||
"RootBlogName" TEXT,
|
||||
"PostID" INTEGER,
|
||||
"NoteBlogName" TEXT,
|
||||
"TimeStamp" INTEGER,
|
||||
"Type" TEXT,
|
||||
"replyText" TEXT DEFAULT '.',
|
||||
"DatetimeCrawled" TEXT DEFAULT '2/12/26 12am',
|
||||
"DateModified" TEXT,
|
||||
"DateCreated" TEXT,
|
||||
"RootBlogName" TEXT,
|
||||
"PostID" INTEGER,
|
||||
"NoteBlogName" TEXT,
|
||||
"TimeStamp" INTEGER,
|
||||
"Type" TEXT,
|
||||
"replyText" TEXT DEFAULT '.',
|
||||
"DatetimeCrawled" TEXT DEFAULT '2/12/26 12am',
|
||||
"DateModified" TEXT,
|
||||
"DateCreated" TEXT,
|
||||
IsActive INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY("RootBlogName","PostID","TimeStamp","Type","NoteBlogName")
|
||||
);
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE INDEX "Notes_idx_06e01ae3" ON "Notes" ("TimeStamp" DESC);
|
||||
CREATE INDEX "ix_NoteBlogName01" ON "Notes" ("NoteBlogName");
|
||||
CREATE INDEX "ix_NoteBlogName01" ON "Notes" ("NoteBlogName");
|
||||
```
|
||||
|
||||
**`WITHOUT ROWID`, since 2026-08-07.** The rows live in the primary key's b-tree
|
||||
rather than in a rowid table with a separate key index beside it. Nothing about the
|
||||
SQL surface changes — same columns, same types, same constraint — but two
|
||||
consequences are worth knowing before adding an index here:
|
||||
|
||||
- There is no `rowid` on this table. `SELECT rowid FROM Notes` is an error, and no
|
||||
code in any of the three apps relied on it.
|
||||
- A secondary index carries the whole five-column primary key as its row reference
|
||||
instead of a compact rowid, so indexes on this table are **expensive**.
|
||||
`ix_NoteBlogName01` costs 58 MB, up from 25 MB before the conversion. It earns
|
||||
that: Rolodex filters on `NoteBlogName` and the crawler joins on it.
|
||||
|
||||
**`Notes_idx_06e01ae3` on `TimeStamp DESC` was dropped at the same time.** It cost
|
||||
14 MB as a rowid index and would have cost 58 MB after the conversion. It was worth
|
||||
neither: the crawler's only `TimeStamp` filter (`>= 1535778000`) excludes 786 rows
|
||||
of 1.18M, Rolodex's default Notes sort carries a three-column tiebreaker that forces
|
||||
a full sort regardless, and the reply-matching `UPDATE` uses `ABS(TimeStamp - ?) <= 5`,
|
||||
which no index on `TimeStamp` can serve. The one path that got slower is Rolodex's
|
||||
Notes page with a date-range filter: 60 ms to 164 ms.
|
||||
|
||||
See `../shrink-db.sql` for the full rationale and the applied result.
|
||||
|
||||
**`DatetimeCrawled` is `NULL` on 1,148,077 rows, and that is the honest value.** Those
|
||||
rows previously stored the literal string `'2/12/26 12am'` — this column's own DDL
|
||||
default, written as a bulk backfill placeholder rather than as a crawl time. They were
|
||||
set to `NULL` on 2026-08-07, which is what consumers already displayed them as: the
|
||||
string parses as a date in neither format this schema writes.
|
||||
|
||||
Note the trap: **the `DEFAULT '2/12/26 12am'` clause is still in the DDL above.** Any
|
||||
`INSERT` that omits this column writes the placeholder straight back. The crawler names
|
||||
it explicitly on every insert, so nothing reintroduces it today, but a new writer that
|
||||
forgets to would — which is why consumers should keep treating an unparseable value here
|
||||
as "unknown" rather than assuming `NULL` is now the only such marker.
|
||||
|
||||
One row per engagement event. `TimeStamp` is **unix seconds** — unlike every date column
|
||||
elsewhere in the schema, which are text.
|
||||
|
||||
@@ -155,8 +189,11 @@ At 1.19M rows this is the table that dictates how the whole database has to be q
|
||||
- The only fast access paths are the primary key's leading columns (`RootBlogName`, then
|
||||
`PostID`) and `ix_NoteBlogName01` on `NoteBlogName`. "Notes received by a blog" and
|
||||
"notes given by a blog" are both cheap; almost nothing else is.
|
||||
- Ordering by anything but `TimeStamp` is a full sort of whatever the filters leave.
|
||||
- `replyText` is `'.'` on 1,174,706 rows — only `reply` notes carry real text.
|
||||
- **Every** ordering here is a full sort of whatever the filters leave, `TimeStamp`
|
||||
included. That was already true in practice of the default `TimeStamp` order, whose
|
||||
tiebreakers forced a sort even while `Notes_idx_06e01ae3` existed; since that index
|
||||
was dropped on 2026-08-07 it is true unconditionally. Filter first, then sort.
|
||||
- `replyText` is `'.'` on 1,167,464 rows — only `reply` notes carry real text.
|
||||
|
||||
### Referential integrity
|
||||
|
||||
@@ -178,7 +215,7 @@ consumer.
|
||||
|
||||
| Column | `'.'` rows |
|
||||
|---|--:|
|
||||
| `Notes.replyText` | 1,174,706 |
|
||||
| `Notes.replyText` | 1,167,464 |
|
||||
| `Posts.Title` | 13,144 |
|
||||
| `Posts.Body` | 172 |
|
||||
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
-- shrink-db.sql
|
||||
-- Reduces TL.db from ~267 MB to ~207 MB (-22%) with no application changes,
|
||||
-- and no visible change in any of the three apps that touch this file.
|
||||
--
|
||||
-- The three consumers, and what each one uses:
|
||||
-- URLNotesGrabberCORE System.Data.SQLite 1.0.119 writes Notes, Posts, Blogs
|
||||
-- Rolodex (web) Microsoft.Data.Sqlite 10.0 reads all three; soft-deletes via IsActive
|
||||
-- TumblThree System.Data.SQLite.Core 1.0.119
|
||||
-- one statement only, ManagerController.cs:972 --
|
||||
-- "UPDATE Blogs SET IsActive = 0, DateModified = @DateModified
|
||||
-- WHERE BlogName = @BlogName"
|
||||
-- Nothing below touches the Blogs table, so TumblThree is
|
||||
-- unaffected. (Its GlobalDatabaseService talks to TumblThree's
|
||||
-- own separate FileEntries/BlogFiles database, not this file.)
|
||||
--
|
||||
-- WITHOUT ROWID needs SQLite >= 3.8.2 (Dec 2013). All three providers above are
|
||||
-- 2024-25 builds, an order of magnitude newer, so STEP 2 is readable by all of them.
|
||||
--
|
||||
-- Every figure below was measured on a copy of the live 267 MB file, and the
|
||||
-- result was checked against all three apps' access patterns:
|
||||
-- PRAGMA integrity_check ....... ok
|
||||
-- row counts ................... Notes 1182333, Posts 22468, Blogs 188620 (unchanged)
|
||||
-- Rolodex soft-delete UPDATE ... works
|
||||
-- Rolodex NoteBlogName filter .. still uses ix_NoteBlogName01
|
||||
-- crawler INSERT OR IGNORE ..... still dedupes (0 dupes admitted)
|
||||
-- TumblThree's UPDATE Blogs ..... untouched -- Blogs is not modified by this script
|
||||
--
|
||||
-- HOW TO RUN (DB Browser for SQLite):
|
||||
-- 1. Stop ALL THREE apps: the crawler, the Rolodex web app, and TumblThree.
|
||||
-- Rolodex holds the file open and checkpoints the WAL, so it must be down,
|
||||
-- not just idle. TumblThree only opens the file for an instant when you
|
||||
-- delete a blog, but it can also launch the crawler on its own
|
||||
-- (UrlNotesGrabberService) -- so close it rather than merely avoiding it.
|
||||
-- 2. Back up TL.db (copy the 267 MB file somewhere safe).
|
||||
-- 3. Open TL.db, go to Execute SQL, paste STEP 1-3, run.
|
||||
-- 4. Click "Write Changes".
|
||||
-- 5. Run Tools > Compact Database. This is VACUUM; it will not run from the
|
||||
-- Execute SQL tab because DB Browser keeps a transaction open there.
|
||||
-- NOTHING SHRINKS ON DISK UNTIL THIS FINISHES.
|
||||
--
|
||||
-- Expected: steps 1-3 a couple of minutes, Compact a couple more.
|
||||
-- Free disk needed during Compact: ~270 MB for the temp copy.
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- STEP 1: drop the TimeStamp index (required by STEP 2, not optional)
|
||||
--------------------------------------------------------------------------
|
||||
-- Measured cost/benefit:
|
||||
--
|
||||
-- * Rolodex's DEFAULT Notes view does not use it. Its sort carries the
|
||||
-- tiebreaker "RootBlogName, PostID, NoteBlogName", which forces a full sort
|
||||
-- regardless -- the query plan is byte-identical with and without the index.
|
||||
-- Sorting.cs:138 already assumes as much, and is right in practice.
|
||||
-- * The crawler's collect query (DataAccess.cs:1183) filters
|
||||
-- TimeStamp >= 1535778000, which excludes 786 of 1,182,333 rows (0.07%).
|
||||
-- A full index scan wearing a disguise. Same measured time without it.
|
||||
-- * The reply-matching UPDATE uses ABS(TimeStamp - ?) <= 5, which can never
|
||||
-- use an index on TimeStamp.
|
||||
-- * It DOES help exactly one path: Rolodex's Notes page with a date-range
|
||||
-- filter applied. 60 ms -> 164 ms. That is the whole of what is lost.
|
||||
--
|
||||
-- And it must go, because after STEP 2 it stops being cheap. A secondary index
|
||||
-- on a WITHOUT ROWID table carries the full 5-column primary key instead of a
|
||||
-- compact rowid, so this index grows 14 MB -> 58 MB. Keeping it lands the file
|
||||
-- at 265 MB instead of 207 MB -- i.e. it cancels the entire exercise to save
|
||||
-- 100 ms on one filtered view.
|
||||
|
||||
DROP INDEX IF EXISTS Notes_idx_06e01ae3;
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- STEP 2: rebuild Notes as WITHOUT ROWID (-32 MB)
|
||||
--------------------------------------------------------------------------
|
||||
-- Notes has a 5-column composite primary key. In a rowid table SQLite stores
|
||||
-- that key twice: once in the table, once in sqlite_autoindex_Notes_1 (62 MB).
|
||||
-- WITHOUT ROWID stores the rows *in* the key's b-tree, so the copy disappears.
|
||||
--
|
||||
-- ix_NoteBlogName01 grows 25 -> 58 MB for the reason described above. Net -32 MB.
|
||||
-- It is kept because Rolodex filters on NoteBlogName and the crawler joins on it.
|
||||
--
|
||||
-- Safe: neither codebase references rowid on Notes (grep across both trees,
|
||||
-- zero matches). IsActive keeps its exact current declaration, which is what
|
||||
-- Rolodex's ActiveFlag predicate reads.
|
||||
|
||||
PRAGMA foreign_keys = off;
|
||||
|
||||
CREATE TABLE Notes_new (
|
||||
"RootBlogName" TEXT,
|
||||
"PostID" INTEGER,
|
||||
"NoteBlogName" TEXT,
|
||||
"TimeStamp" INTEGER,
|
||||
"Type" TEXT,
|
||||
"replyText" TEXT DEFAULT '.',
|
||||
"DatetimeCrawled" TEXT DEFAULT '2/12/26 12am',
|
||||
"DateModified" TEXT,
|
||||
"DateCreated" TEXT,
|
||||
IsActive INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY("RootBlogName","PostID","TimeStamp","Type","NoteBlogName")
|
||||
) WITHOUT ROWID;
|
||||
|
||||
INSERT INTO Notes_new
|
||||
SELECT RootBlogName, PostID, NoteBlogName, TimeStamp, Type,
|
||||
replyText, DatetimeCrawled, DateModified, DateCreated, IsActive
|
||||
FROM Notes;
|
||||
|
||||
DROP TABLE Notes;
|
||||
ALTER TABLE Notes_new RENAME TO Notes;
|
||||
|
||||
CREATE INDEX "ix_NoteBlogName01" ON "Notes" ("NoteBlogName");
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- STEP 3: clear the DatetimeCrawled placeholder (-13 MB)
|
||||
--------------------------------------------------------------------------
|
||||
-- 1,148,077 of 1,182,333 rows hold the literal DDL default '2/12/26 12am' --
|
||||
-- a backfill placeholder, not a crawl time. SQLite stores all 12 bytes of it
|
||||
-- on every one of those rows.
|
||||
--
|
||||
-- This is UI-NEUTRAL in Rolodex, which is why it is safe despite Rolodex
|
||||
-- displaying the column. Rolodex reads and sorts it through DateSql.Sortable
|
||||
-- (DateRange.cs:67), whose CASE matches '____-__-__%' or the 8-character
|
||||
-- '__/__/__'. The 12-character '2/12/26 12am' matches neither, so Sortable
|
||||
-- already returns NULL for these rows and the page already renders an em dash
|
||||
-- and sorts them to the bottom. RolodexRepository.cs:957-963 documents exactly
|
||||
-- this. Writing a real NULL changes the bytes on disk, not the screen.
|
||||
--
|
||||
-- The crawler never reads the column back -- it only writes it on INSERT
|
||||
-- (DataAccess.cs:713, 721).
|
||||
|
||||
UPDATE Notes SET DatetimeCrawled = NULL WHERE DatetimeCrawled = '2/12/26 12am';
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- DELIBERATELY NOT DONE: nulling Notes.DateCreated
|
||||
--------------------------------------------------------------------------
|
||||
-- An earlier draft of this script also cleared DateCreated = '2026-04-13'
|
||||
-- (a further -12 MB). Do not. Unlike DatetimeCrawled, that value DOES match
|
||||
-- Sortable's '____-__-__%' branch, so Rolodex renders it as a real date in the
|
||||
-- "Created" column on the Notes page and Post detail, and sorts by it. Nulling
|
||||
-- it would turn visible dates into em dashes and move rows in the sort order.
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- STEP 4: Write Changes, then Tools > Compact Database
|
||||
--------------------------------------------------------------------------
|
||||
-- Nothing above reclaims disk until VACUUM runs. From the sqlite3 CLI instead:
|
||||
-- sqlite3 TL.db "VACUUM;"
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- VERIFY (run after compacting; file should be ~207 MB)
|
||||
--------------------------------------------------------------------------
|
||||
-- PRAGMA integrity_check;
|
||||
--
|
||||
-- SELECT 'Notes' t, COUNT(*) n FROM Notes
|
||||
-- UNION ALL SELECT 'Posts', COUNT(*) FROM Posts
|
||||
-- UNION ALL SELECT 'Blogs', COUNT(*) FROM Blogs;
|
||||
-- -- expect 1182333 / 22468 / 188620, unchanged
|
||||
--
|
||||
-- SELECT name, SUM(pgsize)/1024/1024 AS mb
|
||||
-- FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC;
|
||||
-- -- expect Notes 78, ix_NoteBlogName01 58, Posts 50, Blogs 14
|
||||
--
|
||||
-- APPLIED 2026-08-07. Actual result: 267.32 MB -> 207.17 MB, integrity_check ok,
|
||||
-- row counts unchanged, journal_mode still wal. VACUUM took 6 seconds.
|
||||
Reference in New Issue
Block a user