Serialize and de-serialize non-keyword schema values correctly. Fixes #52.

This commit is contained in:
Nick Alexander 2016-09-06 12:03:03 -07:00
parent 562ced372d
commit 659c26339c
6 changed files with 80 additions and 29 deletions

View file

@ -656,12 +656,21 @@
(<apply-db-install-assertions [db fragment merge] (<apply-db-install-assertions [db fragment merge]
(go-pair (go-pair
(let [exec (partial s/execute! (:sqlite-connection db))] (let [schema (.-schema db)
->SQLite (partial ds/->SQLite schema)
exec (partial s/execute! (:sqlite-connection db))]
;; TODO: batch insert. ;; TODO: batch insert.
(doseq [[ident attr-map] fragment] (doseq [[ident attr-map] fragment]
(doseq [[attr value] attr-map] (doseq [[attr value] attr-map]
;; This is a little sloppy. We need to store idents as entids, since they're (mostly)
;; :db.type/ref attributes. So we use that entid passes through idents it doesn't
;; recognize, and assuming that we have no :db.type/keyword values that match idents.
;; This is safe for now.
(let [[v tag] (->SQLite (entid db attr) (entid db value))]
(<? (exec (<? (exec
["INSERT INTO schema VALUES (?, ?, ?)" (sqlite-schema/->SQLite ident) (sqlite-schema/->SQLite attr) (sqlite-schema/->SQLite value)]))))) ["INSERT INTO schema VALUES (?, ?, ?, ?)"
(sqlite-schema/->SQLite ident) (sqlite-schema/->SQLite attr)
v tag]))))))
(let [symbolic-schema (merge-with merge (:symbolic-schema db) fragment) (let [symbolic-schema (merge-with merge (:symbolic-schema db) fragment)
schema (ds/schema (into {} (map (fn [[k v]] [(entid db k) v]) symbolic-schema)))] schema (ds/schema (into {} (map (fn [[k v]] [(entid db k) v]) symbolic-schema)))]

View file

@ -28,10 +28,7 @@
Returns a map (keyword ident) -> (integer entid), like {:db/ident 0}." Returns a map (keyword ident) -> (integer entid), like {:db/ident 0}."
(go-pair (go-pair
(let [rows (<? (->> (let [rows (<? (s/all-rows sqlite-connection ["SELECT ident, entid FROM idents"]))]
{:select [:ident :entid] :from [:idents]}
(s/format)
(s/all-rows sqlite-connection)))]
(into {} (map (fn [row] [(sqlite-schema/<-SQLite :db.type/keyword (:ident row)) (:entid row)])) rows)))) (into {} (map (fn [row] [(sqlite-schema/<-SQLite :db.type/keyword (:ident row)) (:entid row)])) rows))))
(defn <parts [sqlite-connection] (defn <parts [sqlite-connection]
@ -39,32 +36,35 @@
Returns a map (keyword part) -> {:start integer :idx integer}, like {:db.part/user {start: 0x100 idx: 0x101}}." Returns a map (keyword part) -> {:start integer :idx integer}, like {:db.part/user {start: 0x100 idx: 0x101}}."
(go-pair (go-pair
(let [rows (<? (->> (let [rows (<? (s/all-rows sqlite-connection ["SELECT part, start, idx FROM parts"]))]
{:select [:part :start :idx] :from [:parts]}
(s/format)
(s/all-rows sqlite-connection)))]
(into {} (map (fn [row] [(sqlite-schema/<-SQLite :db.type/keyword (:part row)) (select-keys row [:start :idx])])) rows)))) (into {} (map (fn [row] [(sqlite-schema/<-SQLite :db.type/keyword (:part row)) (select-keys row [:start :idx])])) rows))))
(defn <symbolic-schema [sqlite-connection] (defn <symbolic-schema [sqlite-connection idents]
"Read the schema map materialized view from the given SQLite store. "Read the schema map materialized view from the given SQLite store.
Returns a map (keyword ident) -> (map (keyword attribute -> keyword value)), like Returns a map (keyword ident) -> (map (keyword attribute -> keyword value)), like
{:db/ident {:db/cardinality :db.cardinality/one}}." {:db/ident {:db/cardinality :db.cardinality/one}}."
(go-pair (go-pair
(let [ident-map (clojure.set/map-invert idents)
ref-tag (sqlite-schema/->tag :db.type/ref)
kw<-SQLite (partial sqlite-schema/<-SQLite :db.type/keyword)]
(->> (->>
(->> (s/all-rows sqlite-connection ["SELECT ident, attr, value, value_type_tag FROM schema"])
{:select [:ident :attr :value] :from [:schema]}
(s/format)
(s/all-rows sqlite-connection))
(<?) (<?)
(group-by (comp (partial sqlite-schema/<-SQLite :db.type/keyword) :ident)) (group-by (comp kw<-SQLite :ident))
(map (fn [[ident rows]] (map (fn [[ident rows]]
[ident [ident
(into {} (map (fn [row] (into {} (map (fn [row]
[(sqlite-schema/<-SQLite :db.type/keyword (:attr row)) (let [tag (:value_type_tag row)
(sqlite-schema/<-SQLite :db.type/keyword (:value row))]) rows))])) ;; TODO: this is wrong, it doesn't handle true. ;; We want a symbolic schema, but most of our values are
(into {})))) ;; :db.type/ref attributes. Map those entids back to idents.
;; This is ad-hoc since we haven't built a functional DB
;; instance yet.
v (if (= tag ref-tag) (get ident-map (:value row)) (:value row))]
[(kw<-SQLite (:attr row))
(sqlite-schema/<-tagged-SQLite tag v)])) rows))]))
(into {})))))
(defn <initialize-connection [sqlite-connection] (defn <initialize-connection [sqlite-connection]
(go-pair (go-pair
@ -121,7 +121,7 @@
;; We just bootstrapped, or we are returning to an already bootstrapped DB. ;; We just bootstrapped, or we are returning to an already bootstrapped DB.
(let [idents (<? (<idents sqlite-connection)) (let [idents (<? (<idents sqlite-connection))
parts (<? (<parts sqlite-connection)) parts (<? (<parts sqlite-connection))
symbolic-schema (<? (<symbolic-schema sqlite-connection))] symbolic-schema (<? (<symbolic-schema sqlite-connection idents))]
(when-not bootstrapped? (when-not bootstrapped?
(when (not (= idents bootstrap/idents)) (when (not (= idents bootstrap/idents))
(raise "After bootstrapping database, expected new materialized idents and old bootstrapped idents to be identical" (raise "After bootstrapping database, expected new materialized idents and old bootstrapped idents to be identical"

View file

@ -57,6 +57,13 @@
:cljs [^boolean unique-value?]) [schema attr] :cljs [^boolean unique-value?]) [schema attr]
(is-attr? schema attr :db.unique/value)) (is-attr? schema attr :db.unique/value))
(defn doc [schema attr]
(get-in (.-schema schema) [attr :db/doc]))
(defn valueType [schema attr]
(let [schema (.-schema schema)]
(get-in schema [attr :db/valueType])))
(defn schema? [x] (defn schema? [x]
(satisfies? ISchema x)) (satisfies? ISchema x))

View file

@ -109,9 +109,9 @@
;; Materialized views of the schema. ;; Materialized views of the schema.
"CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)" "CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"
;; TODO: allow arbitrary schema values (true/false) and tag the resulting values. "CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value BLOB NOT NULL, value_type_tag SMALLINT NOT NULL,
"CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (ident) REFERENCES idents (ident))" FOREIGN KEY (ident) REFERENCES idents (ident))"
"CREATE INDEX idx_schema_unique ON schema (ident, attr, value)" "CREATE INDEX idx_schema_unique ON schema (ident, attr, value, value_type_tag)"
"CREATE TABLE parts (part TEXT NOT NULL PRIMARY KEY, start INTEGER NOT NULL, idx INTEGER NOT NULL)" "CREATE TABLE parts (part TEXT NOT NULL PRIMARY KEY, start INTEGER NOT NULL, idx INTEGER NOT NULL)"
]) ])

View file

@ -19,7 +19,7 @@
;; :db/cardinality :db.cardinality/many} ;; :db/cardinality :db.cardinality/many}
:db/txInstant {:db/valueType :db.type/long :db/txInstant {:db/valueType :db.type/long
:db/cardinality :db.cardinality/one :db/cardinality :db.cardinality/one
} ;; :db/index true} TODO: Handle this using SQLite protocol. :db/index true}
:db/valueType {:db/valueType :db.type/ref :db/valueType {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one} :db/cardinality :db.cardinality/one}
:db/cardinality {:db/valueType :db.type/ref :db/cardinality {:db/valueType :db.type/ref

View file

@ -781,4 +781,39 @@
;; [eid2 :test/ref eid1] is gone, since the ref eid1 is gone. ;; [eid2 :test/ref eid1] is gone, since the ref eid1 is gone.
#{})))))))) #{}))))))))
;; We don't use deftest-db in order to be able to re-open an on disk file.
(deftest-async test-reopen-schema
(with-tempfile [t (tempfile)]
(let [conn (<? (d/<connect t))
test-schema [{:db/id (d/id-literal :db.part/user -1)
:db/ident :test/fulltext
:db/cardinality :db.cardinality/many
:db/valueType :db.type/string
:db/fulltext true
:db/doc "Documentation string"
:db.install/_attribute :db.part/db}]
{tx0 :tx} (<? (d/<transact! conn test-schema))]
(testing "Values in schema are correct initially"
(let [db (d/db conn)
schema (d/schema db)]
(is (= true (ds/indexing? schema (d/entid db :db/txInstant))))
(is (= true (ds/fulltext? schema (d/entid db :test/fulltext))))
(is (= "Documentation string" (ds/doc schema (d/entid db :test/fulltext))))
(is (= :db.type/string (ds/valueType schema (d/entid db :test/fulltext))))))
;; Close and re-open same DB.
(<? (d/<close conn))
(let [conn (<? (d/<connect t))]
(try
(testing "Boolean values in schema are correct after re-opening"
(let [db (d/db conn)
schema (d/schema db)]
(is (= true (ds/indexing? schema (d/entid db :db/txInstant))))
(is (= true (ds/fulltext? schema (d/entid db :test/fulltext))))
(is (= "Documentation string" (ds/doc schema (d/entid db :test/fulltext))))
(is (= :db.type/string (ds/valueType schema (d/entid db :test/fulltext))))))
(finally
(<? (d/<close conn))))))))
#_ (time (t/run-tests)) #_ (time (t/run-tests))