Tag values with value type tags in SQLite.

This commit is contained in:
Nick Alexander 2016-08-05 18:41:49 -07:00 committed by Richard Newman
parent 4d34c820b8
commit ff9a75ae09
7 changed files with 102 additions and 60 deletions

View file

@ -92,12 +92,14 @@
commit the transaction; otherwise, rollback the transaction. Returns a pair-chan resolving to commit the transaction; otherwise, rollback the transaction. Returns a pair-chan resolving to
the pair-chan returned by `chan-fn`.") the pair-chan returned by `chan-fn`.")
(<eavt (<ea [db e a]
[db pattern]
"Search for datoms using the EAVT index.") "Search for datoms using the EAVT index.")
(<avet (<eav [db e a v]
[db pattern] "Search for datoms using the EAVT index.")
(<av
[db a v]
"Search for datoms using the AVET index.") "Search for datoms using the AVET index.")
(<apply-datoms (<apply-datoms
@ -195,30 +197,41 @@
(:sqlite-connection db) chan-fn)) (:sqlite-connection db) chan-fn))
;; TODO: use q for searching? Have q use this for searching for a single pattern? ;; TODO: use q for searching? Have q use this for searching for a single pattern?
(<eavt [db pattern] (<ea [db e a]
(let [[e a v] pattern (go-pair
v (and v (ds/->SQLite schema a v))] ;; We assume e and a are always given. (->>
{:select [:e :a :v :tx [1 :added]]
:from [:all_datoms]
:where [:and [:= :e e] [:= :a a]]}
(s/format) ;; TODO: format these statements only once.
(s/all-rows (:sqlite-connection db))
(<?)
(mapv (partial row->Datom (.-schema db))))))
(<eav [db e a v]
(let [[v tag] (ds/->SQLite schema a v)]
(go-pair (go-pair
(->> (->>
{:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns. {:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns.
:from [:all_datoms] :from [:all_datoms]
:where (cons :and (map #(vector := %1 %2) [:e :a :v] (take-while (comp not nil?) [e a v])))} ;; Must drop nils. :where [:and [:= :e e] [:= :a a] [:= :value_type_tag tag] [:= :v v]]}
(s/format) (s/format) ;; TODO: format these statements only once.
(s/all-rows (:sqlite-connection db)) (s/all-rows (:sqlite-connection db))
(<?) (<?)
(mapv (partial row->Datom (.-schema db))))))) ;; TODO: understand why (schema db) fails. (mapv (partial row->Datom (.-schema db))))))) ;; TODO: understand why (schema db) fails.
(<avet [db pattern] (<av [db a v]
(let [[a v] pattern (let [[v tag] (ds/->SQLite schema a v)]
v (ds/->SQLite schema a v)]
(go-pair (go-pair
(->> (->>
{:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns. {:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns.
:from [:all_datoms] :from [:all_datoms]
:where [:and [:= :a a] [:= :v v] [:= :index_avet 1]]} :where [:and [:= :index_avet 1] [:= :a a] [:= :value_type_tag tag] [:= :v v]]}
(s/format) (s/format) ;; TODO: format these statements only once.
(s/all-rows (:sqlite-connection db)) (s/all-rows (:sqlite-connection db))
(<?) (<?)
@ -232,18 +245,19 @@
;; TODO: batch insert, batch delete. ;; TODO: batch insert, batch delete.
(doseq [datom datoms] (doseq [datom datoms]
(let [[e a v tx added] datom (let [[e a v tx added] datom
v (ds/->SQLite schema a v) [v tag] (ds/->SQLite schema a v)
fulltext? (ds/fulltext? schema a)] fulltext? (ds/fulltext? schema a)]
;; Append to transaction log. ;; Append to transaction log.
(<? (exec (<? (exec
["INSERT INTO transactions VALUES (?, ?, ?, ?, ?)" e a v tx (if added 1 0)])) ["INSERT INTO transactions VALUES (?, ?, ?, ?, ?, ?)" e a v tx (if added 1 0) tag]))
;; Update materialized datom view. ;; Update materialized datom view.
(if (.-added datom) (if (.-added datom)
(let [v (if fulltext? (let [v (if fulltext?
(<? (<insert-fulltext-value db v)) (<? (<insert-fulltext-value db v))
v)] v)]
(<? (exec (<? (exec
["INSERT INTO datoms VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" e a v tx ["INSERT INTO datoms VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" e a v tx
tag ;; value_type_tag
(ds/indexing? schema a) ;; index_avet (ds/indexing? schema a) ;; index_avet
(ds/ref? schema a) ;; index_vaet (ds/ref? schema a) ;; index_vaet
fulltext? ;; index_fulltext fulltext? ;; index_fulltext
@ -253,9 +267,9 @@
(if fulltext? (if fulltext?
(<? (exec (<? (exec
;; TODO: in the future, purge fulltext values from the fulltext_datoms table. ;; TODO: in the future, purge fulltext values from the fulltext_datoms table.
["DELETE FROM datoms WHERE (e = ? AND a = ? AND v IN (SELECT rowid FROM fulltext_values WHERE text = ?))" e a v])) ["DELETE FROM datoms WHERE (e = ? AND a = ? AND value_type_tag = ? AND v IN (SELECT rowid FROM fulltext_values WHERE text = ?))" e a tag v]))
(<? (exec (<? (exec
["DELETE FROM datoms WHERE (e = ? AND a = ? AND v = ?)" e a v]))))))) ["DELETE FROM datoms WHERE (e = ? AND a = ? AND value_type_tag = ? AND v = ?)" e a tag v])))))))
db)) db))
(<advance-tx [db] (<advance-tx [db]

View file

@ -101,8 +101,8 @@
:db.type/keyword { :valid? keyword? } :db.type/keyword { :valid? keyword? }
:db.type/string { :valid? string? } :db.type/string { :valid? string? }
:db.type/boolean { :valid? #?(:clj #(instance? Boolean %) :cljs #(= js/Boolean (type %))) } :db.type/boolean { :valid? #?(:clj #(instance? Boolean %) :cljs #(= js/Boolean (type %))) }
:db.type/integer { :valid? integer? } :db.type/long { :valid? integer? }
:db.type/real { :valid? #?(:clj float? :cljs number?) } :db.type/double { :valid? #?(:clj float? :cljs number?) }
}) })
(defn #?@(:clj [^Boolean ensure-valid-value] (defn #?@(:clj [^Boolean ensure-valid-value]
@ -125,7 +125,7 @@
(if-let [valueType (get-in schema [attr :db/valueType])] (if-let [valueType (get-in schema [attr :db/valueType])]
(if-let [valid? (get-in value-type-map [valueType :valid?])] (if-let [valid? (get-in value-type-map [valueType :valid?])]
(if (valid? value) (if (valid? value)
(sqlite-schema/->SQLite value) [(sqlite-schema/->SQLite value) (sqlite-schema/->tag valueType)]
(raise "Invalid value for attribute " attr ", expected " valueType " but got " value (raise "Invalid value for attribute " attr ", expected " valueType " but got " value
{:error :schema/valueType, :attribute attr, :value value})) {:error :schema/valueType, :attribute attr, :value value}))
(raise "Unknown valueType for attribute " attr ", expected one of " (sorted-set (keys value-type-map)) (raise "Unknown valueType for attribute " attr ", expected one of " (sorted-set (keys value-type-map))

View file

@ -8,7 +8,7 @@
[datomish.pair-chan :refer [go-pair <?]] [datomish.pair-chan :refer [go-pair <?]]
[cljs.core.async.macros :refer [go]])) [cljs.core.async.macros :refer [go]]))
(:require (:require
[datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise-str cond-let]] [datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise raise-str cond-let]]
[datomish.sqlite :as s] [datomish.sqlite :as s]
#?@(:clj [[datomish.pair-chan :refer [go-pair <?]] #?@(:clj [[datomish.pair-chan :refer [go-pair <?]]
[clojure.core.async :refer [go <! >!]]]) [clojure.core.async :refer [go <! >!]]])
@ -17,35 +17,54 @@
(def current-version 1) (def current-version 1)
;; Datomish rows are tagged with a numeric representation of :db/valueType:
;; The tag is used to limit queries, and therefore is placed carefully in the relevant indices to
;; allow searching numeric longs and doubles quickly. The tag is also used to convert SQLite values
;; to the correct Datomish value type on query egress.
(def value-type-tag-map
{:db.type/ref 0
:db.type/boolean 1
:db.type/instant 4
:db.type/long 5 ;; SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
:db.type/double 5 ;; SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
:db.type/string 10
:db.type/uuid 11
:db.type/uri 12
:db.type/keyword 13})
(def v1-statements (def v1-statements
["CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, ["CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL,
value_type_tag SMALLINT NOT NULL,
index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0, index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0,
index_fulltext TINYINT NOT NULL DEFAULT 0, index_fulltext TINYINT NOT NULL DEFAULT 0,
unique_value TINYINT NOT NULL DEFAULT 0, unique_identity TINYINT NOT NULL DEFAULT 0)" unique_value TINYINT NOT NULL DEFAULT 0, unique_identity TINYINT NOT NULL DEFAULT 0)"
"CREATE INDEX idx_datoms_eavt ON datoms (e, a, v)" "CREATE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"
"CREATE INDEX idx_datoms_aevt ON datoms (a, e, v)" "CREATE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"
;; Opt-in index: only if a has :db/index true. ;; Opt-in index: only if a has :db/index true.
"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, v, e) WHERE index_avet IS NOT 0" "CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"
;; Opt-in index: only if a has :db/valueType :db.type/ref. ;; Opt-in index: only if a has :db/valueType :db.type/ref. No need for tag here since all
;; indexed elements are refs.
"CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0" "CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0"
;; Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string, ;; Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string,
;; which is not :db/valueType :db.type/ref. That is, index_vaet and index_fulltext are mutually ;; which is not :db/valueType :db.type/ref. That is, index_vaet and index_fulltext are mutually
;; exclusive. ;; exclusive.
"CREATE INDEX idx_datoms_fulltext ON datoms (v, a, e) WHERE index_fulltext IS NOT 0" "CREATE INDEX idx_datoms_fulltext ON datoms (value_type_tag, v, a, e) WHERE index_fulltext IS NOT 0"
;; TODO: possibly remove this index. :db.unique/value should be asserted by the transactor in ;; TODO: possibly remove this index. :db.unique/value should be asserted by the transactor in
;; all cases, but the index may speed up some of SQLite's query planning. For now, it services ;; all cases, but the index may speed up some of SQLite's query planning. For now, it services
;; to validate the transactor implementation. ;; to validate the transactor implementation. Note that tag is needed here, since we could have
"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (v) WHERE unique_value IS NOT 0" ;; a keyword (stored as ":foo") that overlaps a string value ":foo".
"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (value_type_tag, v) WHERE unique_value IS NOT 0"
;; TODO: possibly remove this index. :db.unique/identity should be asserted by the transactor in ;; TODO: possibly remove this index. :db.unique/identity should be asserted by the transactor in
;; all cases, but the index may speed up some of SQLite's query planning. For now, it serves to ;; all cases, but the index may speed up some of SQLite's query planning. For now, it serves to
;; validate the transactor implementation. ;; validate the transactor implementation. Note that tag is needed here to differentiate, e.g.,
"CREATE UNIQUE INDEX idx_datoms_unique_identity ON datoms (a, v) WHERE unique_identity IS NOT 0" ;; keywords and strings.
"CREATE UNIQUE INDEX idx_datoms_unique_identity ON datoms (a, value_type_tag, v) WHERE unique_identity IS NOT 0"
"CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1)" "CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1, value_type_tag SMALLINT NOT NULL)"
"CREATE INDEX idx_transactions_tx ON transactions (tx)" "CREATE INDEX idx_transactions_tx ON transactions (tx)"
;; Fulltext indexing. ;; Fulltext indexing.
@ -61,21 +80,22 @@
;; A view transparently interpolating fulltext indexed values into the datom structure. ;; A view transparently interpolating fulltext indexed values into the datom structure.
"CREATE VIEW fulltext_datoms AS "CREATE VIEW fulltext_datoms AS
SELECT e, a, fulltext_values.text AS v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity SELECT e, a, fulltext_values.text AS v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
FROM datoms, fulltext_values FROM datoms, fulltext_values
WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid" WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid"
;; A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure. ;; A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure.
"CREATE VIEW all_datoms AS "CREATE VIEW all_datoms AS
SELECT e, a, v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
FROM datoms FROM datoms
WHERE index_fulltext IS 0 WHERE index_fulltext IS 0
UNION ALL UNION ALL
SELECT e, a, v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
FROM fulltext_datoms" FROM fulltext_datoms"
;; Materialized views of the schema. ;; Materialized views of the schema.
"CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)" "CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"
;; TODO: allow arbitrary schema values (true/false) and tag the resulting values.
"CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (ident) REFERENCES idents (ident))" "CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (ident) REFERENCES idents (ident))"
"CREATE INDEX idx_schema_unique ON schema (ident, attr, value)" "CREATE INDEX idx_schema_unique ON schema (ident, attr, value)"
]) ])
@ -142,6 +162,7 @@
Double Double
(->SQLite [x] x)] (->SQLite [x] x)]
:cljs :cljs
[string [string
(->SQLite [x] x) (->SQLite [x] x)
@ -162,5 +183,11 @@
:db.type/keyword (keyword (subs value 1)) :db.type/keyword (keyword (subs value 1))
:db.type/string value :db.type/string value
:db.type/boolean (not= value 0) :db.type/boolean (not= value 0)
:db.type/integer value :db.type/long value
:db.type/real value)) :db.type/double value))
(defn ->tag [valueType]
(or
(valueType value-type-tag-map)
(raise "Unknown valueType " valueType ", expected one of " (sorted-set (keys value-type-tag-map))
{:error :SQLite/tag, :valueType valueType})))

View file

@ -219,7 +219,7 @@
(vec (for [[op & entity] (:entities report)] (vec (for [[op & entity] (:entities report)]
(into [op] (for [field entity] (into [op] (for [field entity]
(if (lookup-ref? field) (if (lookup-ref? field)
(first (<? (db/<eavt db field))) ;; TODO improve this -- this should be avet, shouldn't it? (first (<? (apply db/<av db field)))
field))))) field)))))
(assoc-in report [:entities])))) ;; TODO: meta. (assoc-in report [:entities])))) ;; TODO: meta.
@ -289,7 +289,7 @@
(and (id-literal? e) (and (id-literal? e)
(ds/unique-identity? (db/schema db) a) (ds/unique-identity? (db/schema db) a)
(not-any? id-literal? [a v])) (not-any? id-literal? [a v]))
(let [upserted-eid (:e (first (<? (db/<avet db [a v])))) (let [upserted-eid (:e (first (<? (db/<av db a v))))
allocated-eid (get-in report [:tempids e])] allocated-eid (get-in report [:tempids e])]
(if (and upserted-eid allocated-eid (not= upserted-eid allocated-eid)) (if (and upserted-eid allocated-eid (not= upserted-eid allocated-eid))
(<? (<retry-with-tempid db initial-report initial-entities e upserted-eid)) ;; TODO: not initial report, just the sorted entities here. (<? (<retry-with-tempid db initial-report initial-entities e upserted-eid)) ;; TODO: not initial report, just the sorted entities here.
@ -351,7 +351,7 @@
(when added (when added
;; Check for violated :db/unique constraint between datom and existing store. ;; Check for violated :db/unique constraint between datom and existing store.
(when (ds/unique? schema a) (when (ds/unique? schema a)
(when-let [found (first (<? (db/<avet db [a v])))] (when-let [found (first (<? (db/<av db a v)))]
(raise "Cannot add " datom " because of unique constraint: " found (raise "Cannot add " datom " because of unique constraint: " found
{:error :transact/unique {:error :transact/unique
:attribute a ;; TODO: map attribute back to ident. :attribute a ;; TODO: map attribute back to ident.
@ -401,10 +401,10 @@
(= op :db/add) (= op :db/add)
(if (ds/multival? schema a) (if (ds/multival? schema a)
(if (empty? (<? (db/<eavt db [e a v]))) (if (empty? (<? (db/<eav db e a v)))
(recur (transact-report report (datom e a v tx true)) entities) (recur (transact-report report (datom e a v tx true)) entities)
(recur report entities)) (recur report entities))
(if-let [^Datom old-datom (first (<? (db/<eavt db [e a])))] (if-let [^Datom old-datom (first (<? (db/<ea db e a)))]
(if (= (.-v old-datom) v) (if (= (.-v old-datom) v)
(recur report entities) (recur report entities)
(recur (-> report (recur (-> report
@ -414,7 +414,7 @@
(recur (transact-report report (datom e a v tx true)) entities))) (recur (transact-report report (datom e a v tx true)) entities)))
(= op :db/retract) (= op :db/retract)
(if (first (<? (db/<eavt db [e a v]))) (if (first (<? (db/<eav db e a v)))
(recur (transact-report report (datom e a v tx false)) entities) (recur (transact-report report (datom e a v tx false)) entities)
(recur report entities)) (recur report entities))

View file

@ -17,7 +17,7 @@
;; TODO: support user-specified functions in the future. ;; TODO: support user-specified functions in the future.
;; :db.install/function {:db/valueType :db.type/ref ;; :db.install/function {:db/valueType :db.type/ref
;; :db/cardinality :db.cardinality/many} ;; :db/cardinality :db.cardinality/many}
:db/txInstant {:db/valueType :db.type/integer :db/txInstant {:db/valueType :db.type/long
:db/cardinality :db.cardinality/one :db/cardinality :db.cardinality/one
} ;; :db/index true} TODO: Handle this using SQLite protocol. } ;; :db/index true} TODO: Handle this using SQLite protocol.
:db/valueType {:db/valueType :db.type/ref :db/valueType {:db/valueType :db.type/ref
@ -61,15 +61,16 @@
:db.alter/attribute 22 :db.alter/attribute 22
:db.type/ref 23 :db.type/ref 23
:db.type/keyword 24 :db.type/keyword 24
:db.type/integer 25 ;; TODO: :db.type/long, to match Datomic? :db.type/long 25
:db.type/string 26 :db.type/double 26
:db.type/boolean 27 :db.type/string 27
:db.type/instant 28 :db.type/boolean 28
:db.type/bytes 29 :db.type/instant 29
:db.cardinality/one 30 :db.type/bytes 30
:db.cardinality/many 31 :db.cardinality/one 31
:db.unique/value 32 :db.cardinality/many 32
:db.unique/identity 33}) :db.unique/value 33
:db.unique/identity 34})
(defn tx-data [] (defn tx-data []
(concat (concat

View file

@ -78,7 +78,7 @@
[{:db/id (d/id-literal :test -1) [{:db/id (d/id-literal :test -1)
:db/ident :x :db/ident :x
:db/unique :db.unique/identity :db/unique :db.unique/identity
:db/valueType :db.type/integer} :db/valueType :db.type/long}
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -1)} {:db/id :db.part/db :db.install/attribute (d/id-literal :test -1)}
{:db/id (d/id-literal :test -2) {:db/id (d/id-literal :test -2)
:db/ident :name :db/ident :name
@ -88,7 +88,7 @@
{:db/id (d/id-literal :test -3) {:db/id (d/id-literal :test -3)
:db/ident :y :db/ident :y
:db/cardinality :db.cardinality/many :db/cardinality :db.cardinality/many
:db/valueType :db.type/integer} :db/valueType :db.type/long}
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -3)} {:db/id :db.part/db :db.install/attribute (d/id-literal :test -3)}
{:db/id (d/id-literal :test -5) {:db/id (d/id-literal :test -5)
:db/ident :aka :db/ident :aka
@ -97,7 +97,7 @@
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -5)} {:db/id :db.part/db :db.install/attribute (d/id-literal :test -5)}
{:db/id (d/id-literal :test -6) {:db/id (d/id-literal :test -6)
:db/ident :age :db/ident :age
:db/valueType :db.type/integer} :db/valueType :db.type/long}
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -6)} {:db/id :db.part/db :db.install/attribute (d/id-literal :test -6)}
{:db/id (d/id-literal :test -7) {:db/id (d/id-literal :test -7)
:db/ident :email :db/ident :email

View file

@ -32,7 +32,7 @@
[{:db/id (d/id-literal :test -1) [{:db/id (d/id-literal :test -1)
:db/ident :x :db/ident :x
:db/unique :db.unique/identity :db/unique :db.unique/identity
:db/valueType :db.type/integer :db/valueType :db.type/long
:db.install/_attribute :db.part/db} :db.install/_attribute :db.part/db}
]) ])