greg/mentat
1
0
Fork 0
Code Issues 294 Pull requests Projects Releases 9 Packages Wiki Activity

Tag values with value type tags in SQLite.

Browse source
This commit is contained in:
Nick Alexander 2016-08-05 18:41:49 -07:00 committed by Richard Newman
parent 4d34c820b8
commit ff9a75ae09
7 changed files with 102 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
src/datomish/db.cljc
View file

@ -92,12 +92,14 @@
commit the transaction; otherwise, rollback the transaction. Returns a pair-chan resolving to
the pair-chan returned by `chan-fn`.")
(<eavt
[db pattern]
(<ea [db e a]
"Search for datoms using the EAVT index.")
(<avet
[db pattern]
(<eav [db e a v]
"Search for datoms using the EAVT index.")
(<av
[db a v]
"Search for datoms using the AVET index.")
(<apply-datoms
@ -195,30 +197,41 @@
(:sqlite-connection db) chan-fn))
;; TODO: use q for searching? Have q use this for searching for a single pattern?
(<eavt [db pattern]
(let [[e a v] pattern
v (and v (ds/->SQLite schema a v))] ;; We assume e and a are always given.
(<ea [db e a]
(go-pair
(->>
{:select [:e :a :v :tx [1 :added]]
:from [:all_datoms]
:where [:and [:= :e e] [:= :a a]]}
(s/format) ;; TODO: format these statements only once.
(s/all-rows (:sqlite-connection db))
(<?)
(mapv (partial row->Datom (.-schema db))))))
(<eav [db e a v]
(let [[v tag] (ds/->SQLite schema a v)]
(go-pair
(->>
{:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns.
:from [:all_datoms]
:where (cons :and (map #(vector := %1 %2) [:e :a :v] (take-while (comp not nil?) [e a v])))} ;; Must drop nils.
(s/format)
:where [:and [:= :e e] [:= :a a] [:= :value_type_tag tag] [:= :v v]]}
(s/format) ;; TODO: format these statements only once.
(s/all-rows (:sqlite-connection db))
(<?)
(mapv (partial row->Datom (.-schema db))))))) ;; TODO: understand why (schema db) fails.
(<avet [db pattern]
(let [[a v] pattern
v (ds/->SQLite schema a v)]
(<av [db a v]
(let [[v tag] (ds/->SQLite schema a v)]
(go-pair
(->>
{:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns.
:from [:all_datoms]
:where [:and [:= :a a] [:= :v v] [:= :index_avet 1]]}
(s/format)
:where [:and [:= :index_avet 1] [:= :a a] [:= :value_type_tag tag] [:= :v v]]}
(s/format) ;; TODO: format these statements only once.
(s/all-rows (:sqlite-connection db))
(<?)
@ -232,18 +245,19 @@
;; TODO: batch insert, batch delete.
(doseq [datom datoms]
(let [[e a v tx added] datom
v (ds/->SQLite schema a v)
[v tag] (ds/->SQLite schema a v)
fulltext? (ds/fulltext? schema a)]
;; Append to transaction log.
(<? (exec
["INSERT INTO transactions VALUES (?, ?, ?, ?, ?)" e a v tx (if added 1 0)]))
["INSERT INTO transactions VALUES (?, ?, ?, ?, ?, ?)" e a v tx (if added 1 0) tag]))
;; Update materialized datom view.
(if (.-added datom)
(let [v (if fulltext?
(<? (<insert-fulltext-value db v))
v)]
(<? (exec
["INSERT INTO datoms VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" e a v tx
["INSERT INTO datoms VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" e a v tx
tag ;; value_type_tag
(ds/indexing? schema a) ;; index_avet
(ds/ref? schema a) ;; index_vaet
fulltext? ;; index_fulltext
@ -253,9 +267,9 @@
(if fulltext?
(<? (exec
;; TODO: in the future, purge fulltext values from the fulltext_datoms table.
["DELETE FROM datoms WHERE (e = ? AND a = ? AND v IN (SELECT rowid FROM fulltext_values WHERE text = ?))" e a v]))
["DELETE FROM datoms WHERE (e = ? AND a = ? AND value_type_tag = ? AND v IN (SELECT rowid FROM fulltext_values WHERE text = ?))" e a tag v]))
(<? (exec
["DELETE FROM datoms WHERE (e = ? AND a = ? AND v = ?)" e a v])))))))
["DELETE FROM datoms WHERE (e = ? AND a = ? AND value_type_tag = ? AND v = ?)" e a tag v])))))))
db))
(<advance-tx [db]
@ -270,7 +284,7 @@
(<apply-db-ident-assertions [db added-idents merge]
(go-pair
(let [exec (partial s/execute! (:sqlite-connection db))]
(let [exec (partial s/execute! (:sqlite-connection db))]
;; TODO: batch insert.
(doseq [[ident entid] added-idents]
(<? (exec
@ -282,7 +296,7 @@
(<apply-db-install-assertions [db fragment merge]
(go-pair
(let [exec (partial s/execute! (:sqlite-connection db))]
(let [exec (partial s/execute! (:sqlite-connection db))]
;; TODO: batch insert.
(doseq [[ident attr-map] fragment]
(doseq [[attr value] attr-map]

6
src/datomish/schema.cljc
View file

@ -101,8 +101,8 @@
:db.type/keyword { :valid? keyword? }
:db.type/string { :valid? string? }
:db.type/boolean { :valid? #?(:clj #(instance? Boolean %) :cljs #(= js/Boolean (type %))) }
:db.type/integer { :valid? integer? }
:db.type/real { :valid? #?(:clj float? :cljs number?) }
:db.type/long { :valid? integer? }
:db.type/double { :valid? #?(:clj float? :cljs number?) }
})
(defn #?@(:clj [^Boolean ensure-valid-value]
@ -125,7 +125,7 @@
(if-let [valueType (get-in schema [attr :db/valueType])]
(if-let [valid? (get-in value-type-map [valueType :valid?])]
(if (valid? value)
(sqlite-schema/->SQLite value)
[(sqlite-schema/->SQLite value) (sqlite-schema/->tag valueType)]
(raise "Invalid value for attribute " attr ", expected " valueType " but got " value
{:error :schema/valueType, :attribute attr, :value value}))
(raise "Unknown valueType for attribute " attr ", expected one of " (sorted-set (keys value-type-map))

59
src/datomish/sqlite_schema.cljc
View file

@ -8,7 +8,7 @@
[datomish.pair-chan :refer [go-pair <?]]
[cljs.core.async.macros :refer [go]]))
(:require
[datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise-str cond-let]]
[datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise raise-str cond-let]]
[datomish.sqlite :as s]
#?@(:clj [[datomish.pair-chan :refer [go-pair <?]]
[clojure.core.async :refer [go <! >!]]])
@ -17,35 +17,54 @@
(def current-version 1)
;; Datomish rows are tagged with a numeric representation of :db/valueType:
;; The tag is used to limit queries, and therefore is placed carefully in the relevant indices to
;; allow searching numeric longs and doubles quickly. The tag is also used to convert SQLite values
;; to the correct Datomish value type on query egress.
(def value-type-tag-map
{:db.type/ref 0
:db.type/boolean 1
:db.type/instant 4
:db.type/long 5 ;; SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
:db.type/double 5 ;; SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
:db.type/string 10
:db.type/uuid 11
:db.type/uri 12
:db.type/keyword 13})
(def v1-statements
["CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL,
value_type_tag SMALLINT NOT NULL,
index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0,
index_fulltext TINYINT NOT NULL DEFAULT 0,
unique_value TINYINT NOT NULL DEFAULT 0, unique_identity TINYINT NOT NULL DEFAULT 0)"
"CREATE INDEX idx_datoms_eavt ON datoms (e, a, v)"
"CREATE INDEX idx_datoms_aevt ON datoms (a, e, v)"
"CREATE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"
"CREATE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"
;; Opt-in index: only if a has :db/index true.
"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, v, e) WHERE index_avet IS NOT 0"
"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"
;; Opt-in index: only if a has :db/valueType :db.type/ref.
;; Opt-in index: only if a has :db/valueType :db.type/ref. No need for tag here since all
;; indexed elements are refs.
"CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0"
;; Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string,
;; which is not :db/valueType :db.type/ref. That is, index_vaet and index_fulltext are mutually
;; exclusive.
"CREATE INDEX idx_datoms_fulltext ON datoms (v, a, e) WHERE index_fulltext IS NOT 0"
"CREATE INDEX idx_datoms_fulltext ON datoms (value_type_tag, v, a, e) WHERE index_fulltext IS NOT 0"
;; TODO: possibly remove this index. :db.unique/value should be asserted by the transactor in
;; all cases, but the index may speed up some of SQLite's query planning. For now, it services
;; to validate the transactor implementation.
"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (v) WHERE unique_value IS NOT 0"
;; to validate the transactor implementation. Note that tag is needed here, since we could have
;; a keyword (stored as ":foo") that overlaps a string value ":foo".
"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (value_type_tag, v) WHERE unique_value IS NOT 0"
;; TODO: possibly remove this index. :db.unique/identity should be asserted by the transactor in
;; all cases, but the index may speed up some of SQLite's query planning. For now, it serves to
;; validate the transactor implementation.
"CREATE UNIQUE INDEX idx_datoms_unique_identity ON datoms (a, v) WHERE unique_identity IS NOT 0"
;; validate the transactor implementation. Note that tag is needed here to differentiate, e.g.,
;; keywords and strings.
"CREATE UNIQUE INDEX idx_datoms_unique_identity ON datoms (a, value_type_tag, v) WHERE unique_identity IS NOT 0"
"CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1)"
"CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1, value_type_tag SMALLINT NOT NULL)"
"CREATE INDEX idx_transactions_tx ON transactions (tx)"
;; Fulltext indexing.
@ -61,21 +80,22 @@
;; A view transparently interpolating fulltext indexed values into the datom structure.
"CREATE VIEW fulltext_datoms AS
SELECT e, a, fulltext_values.text AS v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
SELECT e, a, fulltext_values.text AS v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
FROM datoms, fulltext_values
WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid"
;; A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure.
"CREATE VIEW all_datoms AS
SELECT e, a, v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
FROM datoms
WHERE index_fulltext IS 0
UNION ALL
SELECT e, a, v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
FROM fulltext_datoms"
;; Materialized views of the schema.
"CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"
;; TODO: allow arbitrary schema values (true/false) and tag the resulting values.
"CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (ident) REFERENCES idents (ident))"
"CREATE INDEX idx_schema_unique ON schema (ident, attr, value)"
])
@ -142,6 +162,7 @@
Double
(->SQLite [x] x)]
:cljs
[string
(->SQLite [x] x)
@ -162,5 +183,11 @@
:db.type/keyword (keyword (subs value 1))
:db.type/string value
:db.type/boolean (not= value 0)
:db.type/integer value
:db.type/real value))
:db.type/long value
:db.type/double value))
(defn ->tag [valueType]
(or
(valueType value-type-tag-map)
(raise "Unknown valueType " valueType ", expected one of " (sorted-set (keys value-type-tag-map))
{:error :SQLite/tag, :valueType valueType})))

12
src/datomish/transact.cljc
View file

@ -219,7 +219,7 @@
(vec (for [[op & entity] (:entities report)]
(into [op] (for [field entity]
(if (lookup-ref? field)
(first (<? (db/<eavt db field))) ;; TODO improve this -- this should be avet, shouldn't it?
(first (<? (apply db/<av db field)))
field)))))
(assoc-in report [:entities])))) ;; TODO: meta.
@ -289,7 +289,7 @@
(and (id-literal? e)
(ds/unique-identity? (db/schema db) a)
(not-any? id-literal? [a v]))
(let [upserted-eid (:e (first (<? (db/<avet db [a v]))))
(let [upserted-eid (:e (first (<? (db/<av db a v))))
allocated-eid (get-in report [:tempids e])]
(if (and upserted-eid allocated-eid (not= upserted-eid allocated-eid))
(<? (<retry-with-tempid db initial-report initial-entities e upserted-eid)) ;; TODO: not initial report, just the sorted entities here.
@ -351,7 +351,7 @@
(when added
;; Check for violated :db/unique constraint between datom and existing store.
(when (ds/unique? schema a)
(when-let [found (first (<? (db/<avet db [a v])))]
(when-let [found (first (<? (db/<av db a v)))]
(raise "Cannot add " datom " because of unique constraint: " found
{:error :transact/unique
:attribute a ;; TODO: map attribute back to ident.
@ -401,10 +401,10 @@
(= op :db/add)
(if (ds/multival? schema a)
(if (empty? (<? (db/<eavt db [e a v])))
(if (empty? (<? (db/<eav db e a v)))
(recur (transact-report report (datom e a v tx true)) entities)
(recur report entities))
(if-let [^Datom old-datom (first (<? (db/<eavt db [e a])))]
(if-let [^Datom old-datom (first (<? (db/<ea db e a)))]
(if (= (.-v old-datom) v)
(recur report entities)
(recur (-> report
@ -414,7 +414,7 @@
(recur (transact-report report (datom e a v tx true)) entities)))
(= op :db/retract)
(if (first (<? (db/<eavt db [e a v])))
(if (first (<? (db/<eav db e a v)))
(recur (transact-report report (datom e a v tx false)) entities)
(recur report entities))

21
src/datomish/transact/bootstrap.cljc
View file

@ -17,7 +17,7 @@
;; TODO: support user-specified functions in the future.
;; :db.install/function {:db/valueType :db.type/ref
;; :db/cardinality :db.cardinality/many}
:db/txInstant {:db/valueType :db.type/integer
:db/txInstant {:db/valueType :db.type/long
:db/cardinality :db.cardinality/one
} ;; :db/index true} TODO: Handle this using SQLite protocol.
:db/valueType {:db/valueType :db.type/ref
@ -61,15 +61,16 @@
:db.alter/attribute 22
:db.type/ref 23
:db.type/keyword 24
:db.type/integer 25 ;; TODO: :db.type/long, to match Datomic?
:db.type/string 26
:db.type/boolean 27
:db.type/instant 28
:db.type/bytes 29
:db.cardinality/one 30
:db.cardinality/many 31
:db.unique/value 32
:db.unique/identity 33})
:db.type/long 25
:db.type/double 26
:db.type/string 27
:db.type/boolean 28
:db.type/instant 29
:db.type/bytes 30
:db.cardinality/one 31
:db.cardinality/many 32
:db.unique/value 33
:db.unique/identity 34})
(defn tx-data []
(concat

6
test/datomish/db_test.cljc
View file

@ -78,7 +78,7 @@
[{:db/id (d/id-literal :test -1)
:db/ident :x
:db/unique :db.unique/identity
:db/valueType :db.type/integer}
:db/valueType :db.type/long}
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -1)}
{:db/id (d/id-literal :test -2)
:db/ident :name
@ -88,7 +88,7 @@
{:db/id (d/id-literal :test -3)
:db/ident :y
:db/cardinality :db.cardinality/many
:db/valueType :db.type/integer}
:db/valueType :db.type/long}
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -3)}
{:db/id (d/id-literal :test -5)
:db/ident :aka
@ -97,7 +97,7 @@
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -5)}
{:db/id (d/id-literal :test -6)
:db/ident :age
:db/valueType :db.type/integer}
:db/valueType :db.type/long}
{:db/id :db.part/db :db.install/attribute (d/id-literal :test -6)}
{:db/id (d/id-literal :test -7)
:db/ident :email

2
test/datomish/query_test.cljc
View file

@ -32,7 +32,7 @@
[{:db/id (d/id-literal :test -1)
:db/ident :x
:db/unique :db.unique/identity
:db/valueType :db.type/integer
:db/valueType :db.type/long
:db.install/_attribute :db.part/db}
])