Part 1: Separate tx_lookup into tx_lookup_before and tx_lookup_after.
This commit is contained in:
Nick Alexander
parent
a7d6a37cfc
commit
da1250d210
2 changed files with 94 additions and 73 deletions
|
|
@ -172,9 +172,7 @@
|
|||
:table-alias source/gensym-table-alias
|
||||
:make-constraints nil}))
|
||||
|
||||
;; TODO: make this not do the tx_lookup. We could achieve this by having additional special values
|
||||
;; of added0, or by separating the tx_lookup table into before and after tables.
|
||||
(defn- retractAttributes->queries [eas tx]
|
||||
(defn- retractAttributes->queries [oeas tx]
|
||||
(let [where-part
|
||||
"(e = ? AND a = ?)"
|
||||
|
||||
|
|
@ -183,21 +181,23 @@
|
|||
(fn [chunk]
|
||||
(cons
|
||||
(apply str
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag)
|
||||
SELECT e, a, v, ?, 0, value_type_tag, v, value_type_tag
|
||||
"INSERT INTO temp.tx_lookup_after (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag,
|
||||
rid, e, a, v, tx, value_type_tag)
|
||||
SELECT e, a, v, ?, 0 + 2, value_type_tag, v, value_type_tag,
|
||||
rowid, e, a, v, ?, value_type_tag
|
||||
FROM datoms
|
||||
WHERE "
|
||||
(repeater (count chunk)))
|
||||
(cons
|
||||
tx
|
||||
(cons
|
||||
tx
|
||||
(mapcat (fn [[_ e a]]
|
||||
[e a])
|
||||
chunk))))
|
||||
(partition-all (quot (dec max-sql-vars) 2) eas))))
|
||||
chunk)))))
|
||||
(partition-all (quot (- max-sql-vars 2) 2) oeas))))
|
||||
|
||||
;; TODO: make this not do the tx_lookup. We could achieve this by having additional special values
|
||||
;; of added0, or by separating the tx_lookup table into before and after tables.
|
||||
(defn- retractEntities->queries [es tx]
|
||||
(defn- retractEntities->queries [oes tx]
|
||||
(let [ref-tag (sqlite-schema/->tag :db.type/ref)
|
||||
|
||||
;; TODO: include index_vaet flag here, so we can use that index to speed up the deletion.
|
||||
|
|
@ -209,27 +209,31 @@
|
|||
(fn [chunk]
|
||||
(cons
|
||||
(apply str
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag)
|
||||
SELECT e, a, v, ?, 0, value_type_tag, v, value_type_tag
|
||||
"INSERT INTO temp.tx_lookup_after (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag,
|
||||
rid, e, a, v, tx, value_type_tag)
|
||||
SELECT e, a, v, ?, 0 + 2, value_type_tag, v, value_type_tag,
|
||||
rowid, e, a, v, ?, value_type_tag
|
||||
FROM datoms
|
||||
WHERE "
|
||||
(repeater (count chunk)))
|
||||
(cons
|
||||
tx
|
||||
(cons
|
||||
tx
|
||||
(mapcat (fn [[_ e]]
|
||||
[e e])
|
||||
chunk))))
|
||||
(partition-all (quot (dec max-sql-vars) 2) es))))
|
||||
chunk)))))
|
||||
(partition-all (quot (- max-sql-vars 2) 2) oes))))
|
||||
|
||||
(defn- retractions->queries [retractions tx fulltext? ->SQLite]
|
||||
(let
|
||||
[f-q
|
||||
"WITH vv AS (SELECT rowid FROM fulltext_values WHERE text = ?)
|
||||
INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag)
|
||||
INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag)
|
||||
VALUES (?, ?, (SELECT rowid FROM vv), ?, 0, ?, (SELECT rowid FROM vv), ?)"
|
||||
|
||||
non-f-q
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag)
|
||||
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag)
|
||||
VALUES (?, ?, ?, ?, 0, ?, ?, ?)"]
|
||||
(map
|
||||
(fn [[_ e a v]]
|
||||
|
|
@ -242,7 +246,7 @@
|
|||
retractions)))
|
||||
|
||||
(defn- non-fts-many->queries [ops tx ->SQLite indexing? ref? unique?]
|
||||
(let [q "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
(let [q "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
|
||||
values-part
|
||||
;; e0, a0, v0, tx0, added0, value_type_tag0
|
||||
|
|
@ -290,7 +294,7 @@
|
|||
[(cons
|
||||
(apply
|
||||
str
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
(first-repeater (count chunk)))
|
||||
(mapcat (fn [[_ e a v]]
|
||||
(let [[v tag] (->SQLite a v)]
|
||||
|
|
@ -304,7 +308,7 @@
|
|||
(cons
|
||||
(apply
|
||||
str
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0) VALUES "
|
||||
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0) VALUES "
|
||||
(second-repeater (count chunk)))
|
||||
(mapcat (fn [[_ e a v]]
|
||||
(let [[v tag] (->SQLite a v)]
|
||||
|
|
@ -341,10 +345,10 @@
|
|||
[["INSERT INTO fulltext_values_view (text, searchid) VALUES (?, ?)"
|
||||
v searchid]
|
||||
|
||||
;; Second query: tx_lookup.
|
||||
;; Second query: lookup.
|
||||
[(str
|
||||
"WITH vv(rowid) AS (SELECT rowid FROM fulltext_values WHERE searchid = ?) "
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
"(?, ?, (SELECT rowid FROM vv), ?, 1, ?, ?, ?, 1, ?, (SELECT rowid FROM vv), ?)")
|
||||
searchid
|
||||
e a tx tag
|
||||
|
|
@ -365,10 +369,10 @@
|
|||
[["INSERT INTO fulltext_values_view (text, searchid) VALUES (?, ?)"
|
||||
v searchid]
|
||||
|
||||
;; Second and third queries: tx_lookup.
|
||||
;; Second and third queries: lookup.
|
||||
[(str
|
||||
"WITH vv(rowid) AS (SELECT rowid FROM fulltext_values WHERE searchid = ?) "
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES "
|
||||
"(?, ?, (SELECT rowid FROM vv), ?, 1, ?, ?, ?, 1, ?, (SELECT rowid FROM vv), ?)")
|
||||
searchid
|
||||
e a tx tag
|
||||
|
|
@ -378,7 +382,7 @@
|
|||
tag]
|
||||
|
||||
[(str
|
||||
"INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0) VALUES "
|
||||
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0) VALUES "
|
||||
"(?, ?, (SELECT rowid FROM fulltext_values WHERE searchid = ?), ?, 0, ?)")
|
||||
e a searchid tx tag]]))
|
||||
ops
|
||||
|
|
@ -390,33 +394,43 @@
|
|||
(try
|
||||
(doseq [q queries]
|
||||
(<? (s/execute! conn q)))
|
||||
(catch #?(:clj java.sql.SQLException :cljs js/Error) e
|
||||
(catch #?(:clj java.lang.Exception :cljs js/Error) e
|
||||
(throw (ex-info exception-message {} e))))))
|
||||
|
||||
(defn- -preamble-drop [conn]
|
||||
(let [preamble-drop-index ["DROP INDEX IF EXISTS id_tx_lookup_added"]
|
||||
preamble-delete-tx-lookup ["DELETE FROM tx_lookup"]]
|
||||
(go-pair
|
||||
(p :preamble
|
||||
(doseq [q [preamble-drop-index preamble-delete-tx-lookup]]
|
||||
(<? (s/execute! conn q)))))))
|
||||
(doseq [q [;; XXX ["DROP INDEX IF EXISTS temp.idx_tx_lookup_before_added"]
|
||||
(sqlite-schema/create-temp-tx-lookup-statement "temp.tx_lookup_before")
|
||||
(sqlite-schema/create-temp-tx-lookup-statement "temp.tx_lookup_after")
|
||||
;; TODO: move later, into -build-transaction.
|
||||
;; temp goes on index name, not table name. See http://stackoverflow.com/a/22308016.
|
||||
(sqlite-schema/create-temp-tx-lookup-eavt-statement "temp.idx_tx_lookup_before_eavt" "tx_lookup_before")
|
||||
(sqlite-schema/create-temp-tx-lookup-eavt-statement "temp.idx_tx_lookup_after_eavt" "tx_lookup_after")
|
||||
["DELETE FROM temp.tx_lookup_before"]
|
||||
["DELETE FROM temp.tx_lookup_after"]]]
|
||||
(<? (s/execute! conn q))))))
|
||||
|
||||
(defn- -after-drop [conn]
|
||||
(go-pair
|
||||
(doseq [q [;; The lookup table takes space on disk, so we purge it aggressively.
|
||||
["DROP INDEX IF EXISTS id_tx_lookup_added"]
|
||||
["DELETE FROM tx_lookup"]]]
|
||||
(<? (s/execute! conn q)))))
|
||||
(p :postamble
|
||||
;; TODO: delete tx_lookup_before after filling tx_lookup_after.
|
||||
(doseq [q [;; XXX ["DROP INDEX IF EXISTS temp.idx_tx_lookup_before_added"]
|
||||
["DROP INDEX IF EXISTS temp.idx_tx_lookup_before_eavt"]
|
||||
["DROP INDEX IF EXISTS temp.idx_tx_lookup_after_eavt"]
|
||||
["DELETE FROM temp.tx_lookup_before"]
|
||||
["DELETE FROM temp.tx_lookup_after"]]]
|
||||
(<? (s/execute! conn q))))))
|
||||
|
||||
(defn- -build-transaction [conn tx]
|
||||
(let [build-indices ["CREATE INDEX IF NOT EXISTS idx_tx_lookup_added ON tx_lookup (added0)"]
|
||||
(let [build-indices ["CREATE INDEX IF NOT EXISTS temp.idx_tx_lookup_added ON tx_lookup_before (added0)"]
|
||||
|
||||
;; First is fast, only one table walk: lookup by exact eav.
|
||||
;; Second is slower, but still only one table walk: lookup old value by ea.
|
||||
insert-into-tx-lookup
|
||||
["INSERT INTO tx_lookup
|
||||
["INSERT INTO temp.tx_lookup_after
|
||||
SELECT t.e0, t.a0, t.v0, t.tx0, t.added0 + 2, t.value_type_tag0, t.index_avet0, t.index_vaet0, t.index_fulltext0, t.unique_value0, t.sv, t.svalue_type_tag, d.rowid, d.e, d.a, d.v, d.tx, d.value_type_tag
|
||||
FROM tx_lookup AS t
|
||||
FROM temp.tx_lookup_before AS t
|
||||
LEFT JOIN datoms AS d
|
||||
ON t.e0 = d.e AND
|
||||
t.a0 = d.a AND
|
||||
|
|
@ -426,7 +440,7 @@
|
|||
|
||||
UNION ALL
|
||||
SELECT t.e0, t.a0, t.v0, t.tx0, t.added0 + 2, t.value_type_tag0, t.index_avet0, t.index_vaet0, t.index_fulltext0, t.unique_value0, t.sv, t.svalue_type_tag, d.rowid, d.e, d.a, d.v, d.tx, d.value_type_tag
|
||||
FROM tx_lookup AS t,
|
||||
FROM temp.tx_lookup_before AS t,
|
||||
datoms AS d
|
||||
WHERE t.sv IS NULL AND
|
||||
t.e0 = d.e AND
|
||||
|
|
@ -435,14 +449,14 @@
|
|||
t-datoms-not-already-present
|
||||
["INSERT INTO transactions (e, a, v, tx, added, value_type_tag)
|
||||
SELECT e0, a0, v0, ?, 1, value_type_tag0
|
||||
FROM tx_lookup
|
||||
WHERE added0 IS 3 AND e IS NULL" tx] ;; TODO: get rid of magic value 3.
|
||||
FROM temp.tx_lookup_after
|
||||
WHERE added0 IS 3 AND e IS NULL" tx] ;; TODO: get rid of magic value 3. XXX
|
||||
|
||||
t-retract-datoms-carefully
|
||||
["INSERT INTO transactions (e, a, v, tx, added, value_type_tag)
|
||||
SELECT e, a, v, ?, 0, value_type_tag
|
||||
FROM tx_lookup
|
||||
WHERE added0 IS 2 AND ((sv IS NOT NULL) OR (sv IS NULL AND v0 IS NOT v)) AND v IS NOT NULL" tx] ;; TODO: get rid of magic value 2.
|
||||
FROM temp.tx_lookup_after
|
||||
WHERE added0 IS 2 AND ((sv IS NOT NULL) OR (sv IS NULL AND v0 IS NOT v)) AND v IS NOT NULL" tx] ;; TODO: get rid of magic value 2. XXX
|
||||
]
|
||||
(go-pair
|
||||
(doseq [q [build-indices insert-into-tx-lookup
|
||||
|
|
@ -455,13 +469,13 @@
|
|||
["INSERT INTO datoms (e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value)
|
||||
SELECT e0, a0, v0, ?, value_type_tag0,
|
||||
index_avet0, index_vaet0, index_fulltext0, unique_value0
|
||||
FROM tx_lookup
|
||||
WHERE added0 IS 3 AND e IS NULL" tx] ;; TODO: get rid of magic value 3.
|
||||
FROM temp.tx_lookup_after
|
||||
WHERE added0 IS 3 AND e IS NULL" tx] ;; TODO: get rid of magic value 3. XXX
|
||||
|
||||
;; TODO: retract fulltext datoms correctly.
|
||||
d-retract-datoms-carefully
|
||||
["WITH ids AS (SELECT l.rid FROM tx_lookup AS l WHERE l.added0 IS 2 AND ((l.sv IS NOT NULL) OR (l.sv IS NULL AND l.v0 IS NOT l.v)))
|
||||
DELETE FROM datoms WHERE rowid IN ids" ;; TODO: get rid of magic value 2.
|
||||
["WITH ids AS (SELECT l.rid FROM temp.tx_lookup_after AS l WHERE l.added0 IS 2 AND ((l.sv IS NOT NULL) OR (l.sv IS NULL AND l.v0 IS NOT l.v)))
|
||||
DELETE FROM datoms WHERE rowid IN ids" ;; TODO: get rid of magic value 2. XXX
|
||||
]]
|
||||
(-run-queries conn [d-datoms-not-already-present d-retract-datoms-carefully]
|
||||
"Transaction violates unique constraint")))
|
||||
|
|
|
|||
|
|
@ -26,24 +26,6 @@
|
|||
"CREATE UNIQUE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"
|
||||
"CREATE UNIQUE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"
|
||||
|
||||
;; n.b., v0/value_type_tag0 can be NULL, in which case we look up v from datoms;
|
||||
;; and the datom columns are NULL into the LEFT JOIN fills them in.
|
||||
;; TODO: update comment about sv.
|
||||
"CREATE TABLE tx_lookup (e0 INTEGER NOT NULL, a0 SMALLINT NOT NULL, v0 BLOB NOT NULL, tx0 INTEGER NOT NULL, added0 TINYINT NOT NULL,
|
||||
value_type_tag0 SMALLINT NOT NULL,
|
||||
index_avet0 TINYINT, index_vaet0 TINYINT,
|
||||
index_fulltext0 TINYINT,
|
||||
unique_value0 TINYINT,
|
||||
sv BLOB,
|
||||
svalue_type_tag SMALLINT,
|
||||
rid INTEGER,
|
||||
e INTEGER, a SMALLINT, v BLOB, tx INTEGER, value_type_tag SMALLINT)"
|
||||
|
||||
;; Note that `id_tx_lookup_added` is created and dropped
|
||||
;; after insertion, which makes insertion slightly faster.
|
||||
;; Prevent overlapping transactions. TODO: drop added0?
|
||||
"CREATE UNIQUE INDEX idx_tx_lookup_eavt ON tx_lookup (e0, a0, v0, added0, value_type_tag0) WHERE sv IS NOT NULL"
|
||||
|
||||
;; Opt-in index: only if a has :db/index true.
|
||||
"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"
|
||||
|
||||
|
|
@ -115,6 +97,31 @@
|
|||
"CREATE TABLE parts (part TEXT NOT NULL PRIMARY KEY, start INTEGER NOT NULL, idx INTEGER NOT NULL)"
|
||||
])
|
||||
|
||||
(defn create-temp-tx-lookup-statement [table-name]
|
||||
;; n.b., v0/value_type_tag0 can be NULL, in which case we look up v from datoms;
|
||||
;; and the datom columns are NULL into the LEFT JOIN fills them in.
|
||||
;; TODO: update comment about sv.
|
||||
[(str "CREATE TABLE IF NOT EXISTS " table-name
|
||||
" (e0 INTEGER NOT NULL, a0 SMALLINT NOT NULL, v0 BLOB NOT NULL, tx0 INTEGER NOT NULL, added0 TINYINT NOT NULL,
|
||||
value_type_tag0 SMALLINT NOT NULL,
|
||||
index_avet0 TINYINT, index_vaet0 TINYINT,
|
||||
index_fulltext0 TINYINT,
|
||||
unique_value0 TINYINT,
|
||||
sv BLOB,
|
||||
svalue_type_tag SMALLINT,
|
||||
rid INTEGER,
|
||||
e INTEGER, a SMALLINT, v BLOB, tx INTEGER, value_type_tag SMALLINT)")])
|
||||
|
||||
(defn create-temp-tx-lookup-eavt-statement [idx-name table-name]
|
||||
;; Note that `id_tx_lookup_added` is created and dropped
|
||||
;; after insertion, which makes insertion slightly faster.
|
||||
;; Prevent overlapping transactions. TODO: drop added0?
|
||||
[(str "CREATE UNIQUE INDEX IF NOT EXISTS "
|
||||
idx-name
|
||||
" ON "
|
||||
table-name
|
||||
" (e0, a0, v0, added0, value_type_tag0) WHERE sv IS NOT NULL")])
|
||||
|
||||
(defn <create-current-version
|
||||
[db]
|
||||
(->>
|
||||
|
|
|
|||
Loading…
Reference in a new issue