From d8c976c3ad7c55c779d4d2ee4183ee3089a168dd Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Thu, 4 Aug 2016 17:40:38 -0700 Subject: [PATCH] Follow-up: split the monolith! This was a little more tricky than might be expected because the initialization process uses the transactor to bootstrap the database. Since Clojure doesn't accept mutually recursive modules, this necessitated a third module, namely "db-factory", which uses both "db" and "transact". While I was here, I started an "api" module, to paper over the potentially complicated internal module structure for external consumers. In time, this "api" module may also grow CLJS-specific JS transformations. --- src/datomish/db.cljc | 755 +---------------------------------- src/datomish/db_factory.cljc | 124 ++++++ src/datomish/sqlite.cljc | 10 + src/datomish/transact.cljc | 671 +++++++++++++++++++++++++++++++ test/datomish/api.cljc | 40 ++ test/datomish/db_test.cljc | 445 ++++++++++----------- 6 files changed, 1052 insertions(+), 993 deletions(-) create mode 100644 src/datomish/db_factory.cljc create mode 100644 src/datomish/transact.cljc create mode 100644 test/datomish/api.cljc diff --git a/src/datomish/db.cljc b/src/datomish/db.cljc index f2f0a537..7ee327e3 100644 --- a/src/datomish/db.cljc +++ b/src/datomish/db.cljc @@ -12,8 +12,6 @@ [datomish.query.projection :as projection] [datomish.query.source :as source] [datomish.query :as query] - [honeysql.core :as sql] - [datomish.transact.bootstrap :as bootstrap] [datomish.datom :as dd :refer [datom datom? #?@(:cljs [Datom])]] [datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise raise-str cond-let]] [datomish.schema :as ds] @@ -36,13 +34,6 @@ (uncaughtException [_ thread ex] (println ex "Uncaught exception on" (.getName thread)))))) -;; Setting this to something else will make your output more readable, -;; but not automatically safe for use. -(def sql-quoting-style :ansi) - -(defn- sql-format [args] - (sql/format args :quoting :ansi)) - (defprotocol IClock (now [clock] @@ -152,7 +143,7 @@ {:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns. :from [:all_datoms] :where (cons :and (map #(vector := %1 %2) [:e :a :v] (take-while (comp not nil?) [e a v])))} ;; Must drop nils. - (sql-format) + (s/format) (s/all-rows (:sqlite-connection db)) (TempId part (swap! -id-literal-idx dec)))) - ([part idx] - (->TempId part idx))) - -(defn id-literal? [x] - (and (instance? TempId x))) - -(defrecord TxReport [db-before ;; The DB before the transaction. - db-after ;; The DB after the transaction. - current-tx ;; The tx ID represented by the transaction in this report. - entities ;; The set of entities (like [:db/add e a v tx]) processed. - tx-data ;; The set of datoms applied to the database, like (Datom. e a v tx added). - tempids ;; The map from id-literal -> numeric entid. - added-parts ;; The set of parts added during the transaction via :db.part/db :db.install/part. - added-idents ;; The map of idents -> entid added during the transaction, via e :db/ident ident. - added-attributes ;; The map of schema attributes (ident -> schema fragment) added during the transaction, via :db.part/db :db.install/attribute. - ]) - -(defn- report? [x] - (and (instance? TxReport x))) - -(defonce -eid (atom (- 0x200 1))) - -;; TODO: better here. -(defn- next-eid [db] - (swap! -eid inc)) - -(defn- allocate-eid - [report id-literal eid] - {:pre [(report? report) (id-literal? id-literal) (and (integer? eid) (not (neg? eid)))]} - - (assoc-in report [:tempids id-literal] eid)) - -;; (def data-readers {'db/id id-literal}) - -;; #?(:cljs -;; (doseq [[tag cb] data-readers] (cljs.reader/register-tag-parser! tag cb))) - -;; TODO: implement support for DB parts? -(def tx0 0x2000000) - -(defn (integer entid), like {:db/ident 0}." - - (let [<-SQLite (get-in ds/value-type-map [:db.type/keyword :<-SQLite])] ;; TODO: make this a protocol. - (go-pair - (let [rows (> - {:select [:ident :entid] :from [:idents]} - (sql-format) - (s/all-rows sqlite-connection)))] - (into {} (map (fn [row] [(<-SQLite (:ident row)) (:entid row)])) rows))))) - -(defn (map (keyword attribute -> keyword value)), like - {:db/ident {:db/cardinality :db.cardinality/one}}." - - (let [<-SQLite (get-in ds/value-type-map [:db.type/keyword :<-SQLite])] ;; TODO: make this a protocol. - (go-pair - (->> - (->> - {:select [:ident :attr :value] :from [:schema]} - (sql-format) - (s/all-rows sqlite-connection)) - (= current-tx 0) - current-tx (max current-tx tx0)] - (when-not bootstrapped - ;; We need to bootstrap the DB. - (let [fail-alter-ident (fn [old new] (if-not (= old new) - (raise "Altering idents is not yet supported, got " new " altering existing ident " old - {:error :schema/alter-idents :old old :new new}) - new)) - fail-alter-attr (fn [old new] (if-not (= old new) - (raise "Altering schema attributes is not yet supported, got " new " altering existing schema attribute " old - {:error :schema/alter-schema :old old :new new}) - new))] - (-> (map->DB - {:sqlite-connection sqlite-connection - :idents bootstrap/idents - :symbolic-schema bootstrap/symbolic-schema - :schema (ds/schema (into {} (map (fn [[k v]] [(k bootstrap/idents) v]) bootstrap/symbolic-schema))) ;; TODO: fail if ident missing. - :current-tx current-tx}) - ;; We use DB - {:sqlite-connection sqlite-connection - :idents idents - :symbolic-schema symbolic-schema - :schema (ds/schema (into {} (map (fn [[k v]] [(k idents) v]) symbolic-schema))) ;; TODO: fail if ident missing. - :current-tx (inc current-tx)}))))) - -(defn connection-with-db [db] - (map->Connection {:current-db (atom db)})) - -;; ;; TODO: persist max-tx and max-eid in SQLite. - -(defn maybe-datom->entity [entity] - (cond - (datom? entity) - (-> - (let [[e a v tx added] entity] - (if added - [:db/add [e a v tx]] - [:db/retract [e a v tx]])) - (with-meta (get (meta entity) :source))) - - true - entity)) - -(defn- #?@(:clj [^Boolean reverse-ref?] - :cljs [^boolean reverse-ref?]) [attr] - (if (keyword? attr) - (= \_ (nth (name attr) 0)) - (raise "Bad attribute type: " attr ", expected keyword" - {:error :transact/syntax, :attribute attr}))) - -(defn- reverse-ref [attr] - (if (keyword? attr) - (if (reverse-ref? attr) - (keyword (namespace attr) (subs (name attr) 1)) - (keyword (namespace attr) (str "_" (name attr)))) - (raise "Bad attribute type: " attr ", expected keyword" - {:error :transact/syntax, :attribute attr}))) - -(declare explode-entity) - -(defn- explode-entity-a-v [db entity eid a v] - ;; a should be symbolic at this point. Map it. TODO: use ident/entid to ensure we have a symbolic attr. - (let [reverse? (reverse-ref? a) - straight-a (if reverse? (reverse-ref a) a) - straight-a* (get-in db [:idents straight-a] straight-a) - _ (when (and reverse? (not (ds/ref? (schema db) straight-a*))) - (raise "Bad attribute " a ": reverse attribute name requires {:db/valueType :db.type/ref} in schema" - {:error :transact/syntax, :attribute a, :op entity})) - a* (get-in db [:idents a] a)] - (cond - reverse? - (explode-entity-a-v db entity v straight-a eid) - - (and (map? v) - (not (id-literal? v))) - ;; Another entity is given as a nested map. - (if (ds/ref? (schema db) straight-a*) - (let [other (assoc v (reverse-ref a) eid - ;; TODO: make the new ID have the same part as the original eid. - ;; TODO: make the new ID not show up in the tempids map. (Does Datomic exposed the new ID this way?) - :db/id (id-literal :db.part/user))] - (explode-entity db other)) - (raise "Bad attribute " a ": nested map " v " given but attribute name requires {:db/valueType :db.type/ref} in schema" - {:error :transact/entity-map-type-ref - :op entity })) - - (sequential? v) - (if (ds/multival? (schema db) a*) ;; dm/schema - (mapcat (partial explode-entity-a-v db entity eid a) v) ;; Allow sequences of nested maps, etc. This does mean [[1]] will work. - (raise "Sequential values " v " but attribute " a " is :db.cardinality/one" - {:error :transact/entity-sequential-cardinality-one - :op entity })) - - true - [[:db/add eid a* v]]))) - -(defn- explode-entity [db entity] - (if (map? entity) - (if-let [eid (:db/id entity)] - (mapcat (partial apply explode-entity-a-v db entity eid) (dissoc entity :db/id)) - (raise "Map entity missing :db/id, got " entity - {:error :transact/entity-missing-db-id - :op entity })) - [entity])) - -(defn explode-entities [db entities] - "Explode map shorthand, such as {:db/id e :attr value :_reverse ref}, to a list of vectors, - like [[:db/add e :attr value] [:db/add ref :reverse e]]." - (mapcat (partial explode-entity db) entities)) - -(defn maybe-ident->entid [db [op e a v tx :as orig]] - (let [e (get (idents db) e e) ;; TODO: use ident, entid here. - a (get (idents db) a a) - v (if (ds/kw? (schema db) a) ;; TODO: decide if this is best. We could also check for ref and numeric types. - v - (get (idents db) v v))] - [op e a v tx])) - -(defrecord Transaction [db tempids entities]) - -(defn- tx-entity [db report] - {:pre [(db? db) (report? report)]} - (let [tx (:tx report) - txInstant (:txInstant report)] - ;; n.b., this must not be symbolic -- it's inserted after we map idents -> entids. - [:db/add tx (get-in db [:idents :db/txInstant]) txInstant])) - -(defn ensure-entity-form [[op e a v & rest :as entity]] - (cond - (not (sequential? entity)) - (raise "Bad entity " entity ", should be sequential at this point" - {:error :transact/bad-entity, :entity entity}) - - (not (contains? #{:db/add :db/retract} op)) - (raise "Unrecognized operation " op " expected one of :db/add :db/retract at this point" - {:error :transact/bad-operation :entity entity }) - - (nil? e) - (raise "Bad entity: nil e in " entity - {:error :transact/bad-entity :entity entity }) - - (nil? a) - (raise "Bad entity: nil a in " entity - {:error :transact/bad-entity :entity entity }) - - (nil? v) - (raise "Bad entity: nil v in " entity - {:error :transact/bad-entity :entity entity }) - - (some? rest) - (raise "Bad entity: too long " entity - {:error :transact/bad-entity :entity entity }) - - true - entity)) - -(defn- tx-instant? [db [op e a & _]] - (and (= op :db/add) - (= e (get-in db [:idents :db/tx])) - (= a (get-in db [:idents :db/txInstant])))) - -(defn- update-txInstant [db report] - "Extract [:db/add :db/tx :db/txInstant ...], and update :txInstant with that value." - {:pre [(db? db) (report? report)]} - - ;; TODO: be more efficient here: don't iterate all entities. - (if-let [[_ _ _ txInstant] (first (filter (partial tx-instant? db) (:entities report)))] - (assoc report :txInstant txInstant) - report)) - -(defn preprocess [db report] - {:pre [(db? db) (report? report)]} - - (let [initial-es (or (:entities report) []) - ;; :db/tx is a "dynamic enum ident" that maps to the current transaction ID. This approach - ;; mimics DataScript's :db/current-tx. (We don't follow DataScript because - ;; current-txInstant is awkward.) It's much simpler than Datomic's approach, which appears - ;; to unify all id-literals in :db.part/tx to the current transaction value, but also seems - ;; inconsistent. - tx (:tx report) - db* (assoc-in db [:idents :db/tx] tx)] - (when-not (sequential? initial-es) - (raise "Bad transaction data " initial-es ", expected sequential collection" - {:error :transact/syntax, :tx-data initial-es})) - - ;; TODO: find an approach that generates less garbage. - (-> - report - - ;; Normalize Datoms into :db/add or :db/retract vectors. - (update :entities (partial map maybe-datom->entity)) - - (update :entities (partial explode-entities db)) - - (update :entities (partial map ensure-entity-form)) - - ;; Replace idents with entids where possible, using db* to capture :db/tx. - (update :entities (partial map (partial maybe-ident->entid db*))) - - ;; If an explicit [:db/add :db/tx :db/txInstant] is not given, add one. Use db* to - ;; capture :db/tx. - (update :entities (fn [entities] - (if (first (filter (partial tx-instant? db*) entities)) - entities - (conj entities (tx-entity db report))))) - - ;; Extract the current txInstant for the report. - (->> (update-txInstant db*))))) - -(defn- lookup-ref? [x] - "Return true if `x` is like [:attr value]." - (and (sequential? x) - (= (count x) 2) - (or (keyword? (first x)) - (integer? (first x))))) - +;; TODO: factor this into the overall design. (defn > - (vec (for [[op & entity] (:entities report)] - (into [op] (for [field entity] - (if (lookup-ref? field) - (first ( - report - (assoc-in [:tempids tempid] upserted-eid) - (assoc-in [:entities] es))))) - -(defn- transact-entity [report entity] - (update-in report [:entities] conj entity)) - -(defn datom. - (doseq [[e a v tx added :as datom] (:tx-data report)] - - (when added - ;; Check for violated :db/unique constraint between datom and existing store. - (when (ds/unique? schema a) - (when-let [found (first (tx-data [db report] - {:pre [(db? db) (report? report)]} - (go-pair - (let [initial-report report - {tx :tx} report - schema (schema db)] - (loop [report initial-report - es (:entities initial-report)] - (let [[[op e a v :as entity] & entities] es] - (cond - (nil? entity) - report - - (= op :db/add) - (if (ds/multival? schema a) - (if (empty? ( report - (transact-report (datom e a (.-v old-datom) tx false)) - (transact-report (datom e a v tx true))) - entities)) - (recur (transact-report report (datom e a v tx true)) entities))) - - (= op :db/retract) - (if (first (> - report - (preprocess db) - - (tx-data db) - (schema-fragment datoms)] - (assoc-in report [:added-attributes] schema-fragment))) - -(defn- > - (map->TxReport - {:db-before db - :db-after db - ;; This mimics DataScript. It's convenient to be able to extract the - ;; transaction ID and transaction timestamp directly from the report; Datomic - ;; makes this surprisingly difficult: one needs a :db.part/tx temporary and an - ;; explicit upsert of that temporary. - :tx (current-tx db) - :txInstant (now db) - :entities tx-data - :tx-data [] - :tempids {} - :added-parts {} - :added-idents {} - :added-attributes {} - }) - - ( - db - - ( report - (assoc-in [:db-after] db-after))))) - -(defn- !]]]) + #?@(:cljs [[datomish.pair-chan] + [cljs.core.async :as a :refer [chan !]]])) + #?(:clj + (:import + [datomish.datom Datom]))) + +;; TODO: implement support for DB parts? +(def tx0 0x2000000) + +(defn (integer entid), like {:db/ident 0}." + + (let [<-SQLite (get-in ds/value-type-map [:db.type/keyword :<-SQLite])] ;; TODO: make this a protocol. + (go-pair + (let [rows (> + {:select [:ident :entid] :from [:idents]} + (s/format) + (s/all-rows sqlite-connection)))] + (into {} (map (fn [row] [(<-SQLite (:ident row)) (:entid row)])) rows))))) + +(defn (map (keyword attribute -> keyword value)), like + {:db/ident {:db/cardinality :db.cardinality/one}}." + + (let [<-SQLite (get-in ds/value-type-map [:db.type/keyword :<-SQLite])] ;; TODO: make this a protocol. + (go-pair + (->> + (->> + {:select [:ident :attr :value] :from [:schema]} + (s/format) + (s/all-rows sqlite-connection)) + (= current-tx 0) + current-tx (max current-tx tx0)] + (when-not bootstrapped + ;; We need to bootstrap the DB. + (let [fail-alter-ident (fn [old new] (if-not (= old new) + (raise "Altering idents is not yet supported, got " new " altering existing ident " old + {:error :schema/alter-idents :old old :new new}) + new)) + fail-alter-attr (fn [old new] (if-not (= old new) + (raise "Altering schema attributes is not yet supported, got " new " altering existing schema attribute " old + {:error :schema/alter-schema :old old :new new}) + new))] + (-> (db/map->DB + {:sqlite-connection sqlite-connection + :idents bootstrap/idents + :symbolic-schema bootstrap/symbolic-schema + :schema (ds/schema (into {} (map (fn [[k v]] [(k bootstrap/idents) v]) bootstrap/symbolic-schema))) ;; TODO: fail if ident missing. + :current-tx current-tx}) + ;; We use DB + {:sqlite-connection sqlite-connection + :idents idents + :symbolic-schema symbolic-schema + :schema (ds/schema (into {} (map (fn [[k v]] [(k idents) v]) symbolic-schema))) ;; TODO: fail if ident missing. + :current-tx (inc current-tx)}))))) diff --git a/src/datomish/sqlite.cljc b/src/datomish/sqlite.cljc index 003fc07c..cc970c3d 100644 --- a/src/datomish/sqlite.cljc +++ b/src/datomish/sqlite.cljc @@ -3,19 +3,29 @@ ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. (ns datomish.sqlite + (:refer-clojure :exclude [format]) #?(:cljs (:require-macros [datomish.pair-chan :refer [go-pair go-safely ! chan put! take! close!]]) :cljs (:require + [honeysql.core] [datomish.pair-chan] [cljs.core.async :as a :refer [! chan put! take! close!]]))) +;; Setting this to something else will make your output more readable, +;; but not automatically safe for use. +(def sql-quoting-style :ansi) + +(defn format [args] + (honeysql.core/format args :quoting :ansi)) + (defprotocol ISQLiteConnection (-execute! [db sql bindings] diff --git a/src/datomish/transact.cljc b/src/datomish/transact.cljc new file mode 100644 index 00000000..46300b1d --- /dev/null +++ b/src/datomish/transact.cljc @@ -0,0 +1,671 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. + +(ns datomish.transact + #?(:cljs + (:require-macros + [datomish.pair-chan :refer [go-pair !]]]) + #?@(:cljs [[datomish.pair-chan] + [cljs.core.async :as a :refer [chan !]]])) + #?(:clj + (:import + [datomish.datom Datom]))) + +(defprotocol IConnection + (close + [conn] + "Close this connection. Returns a pair channel of [nil error].") + + (db + [conn] + "Get the current DB associated with this connection.") + + (history + [conn] + "Get the full transaction history DB associated with this connection.")) + +(defrecord Connection [current-db] + IConnection + (close [conn] (db/close-db @(:current-db conn))) + + (db [conn] @(:current-db conn)) + + (history [conn] + (raise "Datomic's history is not yet supported." {}))) + +(defn conn? [x] + (and (satisfies? IConnection x))) + +;; ---------------------------------------------------------------------------- +;; define data-readers to be made available to EDN readers. in CLJS +;; they're magically available. in CLJ, data_readers.clj may or may +;; not work, but you can always simply do +;; +;; (clojure.edn/read-string {:readers datomish/data-readers} "...") +;; + +(defonce -id-literal-idx (atom -1000000)) + +(defrecord TempId [part idx]) + +(defn id-literal + ([part] + (if (sequential? part) + (apply id-literal part) + (->TempId part (swap! -id-literal-idx dec)))) + ([part idx] + (->TempId part idx))) + +(defn id-literal? [x] + (and (instance? TempId x))) + +(defrecord TxReport [db-before ;; The DB before the transaction. + db-after ;; The DB after the transaction. + current-tx ;; The tx ID represented by the transaction in this report. + entities ;; The set of entities (like [:db/add e a v tx]) processed. + tx-data ;; The set of datoms applied to the database, like (Datom. e a v tx added). + tempids ;; The map from id-literal -> numeric entid. + added-parts ;; The set of parts added during the transaction via :db.part/db :db.install/part. + added-idents ;; The map of idents -> entid added during the transaction, via e :db/ident ident. + added-attributes ;; The map of schema attributes (ident -> schema fragment) added during the transaction, via :db.part/db :db.install/attribute. + ]) + +(defn- report? [x] + (and (instance? TxReport x))) + +(defonce -eid (atom (- 0x200 1))) + +;; TODO: better here. +(defn- next-eid [db] + (swap! -eid inc)) + +(defn- allocate-eid + [report id-literal eid] + {:pre [(report? report) (id-literal? id-literal) (and (integer? eid) (not (neg? eid)))]} + + (assoc-in report [:tempids id-literal] eid)) + +;; (def data-readers {'db/id id-literal}) + +;; #?(:cljs +;; (doseq [[tag cb] data-readers] (cljs.reader/register-tag-parser! tag cb))) + +;; TODO: implement support for DB parts? + +(defn connection-with-db [db] + (map->Connection {:current-db (atom db)})) + +;; ;; TODO: persist max-tx and max-eid in SQLite. + +(defn maybe-datom->entity [entity] + (cond + (datom? entity) + (-> + (let [[e a v tx added] entity] + (if added + [:db/add [e a v tx]] + [:db/retract [e a v tx]])) + (with-meta (get (meta entity) :source))) + + true + entity)) + +(defn- #?@(:clj [^Boolean reverse-ref?] + :cljs [^boolean reverse-ref?]) [attr] + (if (keyword? attr) + (= \_ (nth (name attr) 0)) + (raise "Bad attribute type: " attr ", expected keyword" + {:error :transact/syntax, :attribute attr}))) + +(defn- reverse-ref [attr] + (if (keyword? attr) + (if (reverse-ref? attr) + (keyword (namespace attr) (subs (name attr) 1)) + (keyword (namespace attr) (str "_" (name attr)))) + (raise "Bad attribute type: " attr ", expected keyword" + {:error :transact/syntax, :attribute attr}))) + +(declare explode-entity) + +(defn- explode-entity-a-v [db entity eid a v] + ;; a should be symbolic at this point. Map it. TODO: use ident/entid to ensure we have a symbolic attr. + (let [reverse? (reverse-ref? a) + straight-a (if reverse? (reverse-ref a) a) + straight-a* (get-in db [:idents straight-a] straight-a) + _ (when (and reverse? (not (ds/ref? (db/schema db) straight-a*))) + (raise "Bad attribute " a ": reverse attribute name requires {:db/valueType :db.type/ref} in schema" + {:error :transact/syntax, :attribute a, :op entity})) + a* (get-in db [:idents a] a)] + (cond + reverse? + (explode-entity-a-v db entity v straight-a eid) + + (and (map? v) + (not (id-literal? v))) + ;; Another entity is given as a nested map. + (if (ds/ref? (db/schema db) straight-a*) + (let [other (assoc v (reverse-ref a) eid + ;; TODO: make the new ID have the same part as the original eid. + ;; TODO: make the new ID not show up in the tempids map. (Does Datomic exposed the new ID this way?) + :db/id (id-literal :db.part/user))] + (explode-entity db other)) + (raise "Bad attribute " a ": nested map " v " given but attribute name requires {:db/valueType :db.type/ref} in schema" + {:error :transact/entity-map-type-ref + :op entity })) + + (sequential? v) + (if (ds/multival? (db/schema db) a*) ;; dm/schema + (mapcat (partial explode-entity-a-v db entity eid a) v) ;; Allow sequences of nested maps, etc. This does mean [[1]] will work. + (raise "Sequential values " v " but attribute " a " is :db.cardinality/one" + {:error :transact/entity-sequential-cardinality-one + :op entity })) + + true + [[:db/add eid a* v]]))) + +(defn- explode-entity [db entity] + (if (map? entity) + (if-let [eid (:db/id entity)] + (mapcat (partial apply explode-entity-a-v db entity eid) (dissoc entity :db/id)) + (raise "Map entity missing :db/id, got " entity + {:error :transact/entity-missing-db-id + :op entity })) + [entity])) + +(defn explode-entities [db entities] + "Explode map shorthand, such as {:db/id e :attr value :_reverse ref}, to a list of vectors, + like [[:db/add e :attr value] [:db/add ref :reverse e]]." + (mapcat (partial explode-entity db) entities)) + +(defn maybe-ident->entid [db [op e a v tx :as orig]] + (let [e (get (db/idents db) e e) ;; TODO: use ident, entid here. + a (get (db/idents db) a a) + v (if (ds/kw? (db/schema db) a) ;; TODO: decide if this is best. We could also check for ref and numeric types. + v + (get (db/idents db) v v))] + [op e a v tx])) + +(defrecord Transaction [db tempids entities]) + +(defn- tx-entity [db report] + {:pre [(db/db? db) (report? report)]} + (let [tx (:tx report) + txInstant (:txInstant report)] + ;; n.b., this must not be symbolic -- it's inserted after we map idents -> entids. + [:db/add tx (get-in db [:idents :db/txInstant]) txInstant])) + +(defn ensure-entity-form [[op e a v & rest :as entity]] + (cond + (not (sequential? entity)) + (raise "Bad entity " entity ", should be sequential at this point" + {:error :transact/bad-entity, :entity entity}) + + (not (contains? #{:db/add :db/retract} op)) + (raise "Unrecognized operation " op " expected one of :db/add :db/retract at this point" + {:error :transact/bad-operation :entity entity }) + + (nil? e) + (raise "Bad entity: nil e in " entity + {:error :transact/bad-entity :entity entity }) + + (nil? a) + (raise "Bad entity: nil a in " entity + {:error :transact/bad-entity :entity entity }) + + (nil? v) + (raise "Bad entity: nil v in " entity + {:error :transact/bad-entity :entity entity }) + + (some? rest) + (raise "Bad entity: too long " entity + {:error :transact/bad-entity :entity entity }) + + true + entity)) + +(defn- tx-instant? [db [op e a & _]] + (and (= op :db/add) + (= e (get-in db [:idents :db/tx])) + (= a (get-in db [:idents :db/txInstant])))) + +(defn- update-txInstant [db report] + "Extract [:db/add :db/tx :db/txInstant ...], and update :txInstant with that value." + {:pre [(db/db? db) (report? report)]} + + ;; TODO: be more efficient here: don't iterate all entities. + (if-let [[_ _ _ txInstant] (first (filter (partial tx-instant? db) (:entities report)))] + (assoc report :txInstant txInstant) + report)) + +(defn preprocess [db report] + {:pre [(db/db? db) (report? report)]} + + (let [initial-es (or (:entities report) []) + ;; :db/tx is a "dynamic enum ident" that maps to the current transaction ID. This approach + ;; mimics DataScript's :db/current-tx. (We don't follow DataScript because + ;; current-txInstant is awkward.) It's much simpler than Datomic's approach, which appears + ;; to unify all id-literals in :db.part/tx to the current transaction value, but also seems + ;; inconsistent. + tx (:tx report) + db* (assoc-in db [:idents :db/tx] tx)] + (when-not (sequential? initial-es) + (raise "Bad transaction data " initial-es ", expected sequential collection" + {:error :transact/syntax, :tx-data initial-es})) + + ;; TODO: find an approach that generates less garbage. + (-> + report + + ;; Normalize Datoms into :db/add or :db/retract vectors. + (update :entities (partial map maybe-datom->entity)) + + (update :entities (partial explode-entities db)) + + (update :entities (partial map ensure-entity-form)) + + ;; Replace idents with entids where possible, using db* to capture :db/tx. + (update :entities (partial map (partial maybe-ident->entid db*))) + + ;; If an explicit [:db/add :db/tx :db/txInstant] is not given, add one. Use db* to + ;; capture :db/tx. + (update :entities (fn [entities] + (if (first (filter (partial tx-instant? db*) entities)) + entities + (conj entities (tx-entity db report))))) + + ;; Extract the current txInstant for the report. + (->> (update-txInstant db*))))) + +(defn- lookup-ref? [x] + "Return true if `x` is like [:attr value]." + (and (sequential? x) + (= (count x) 2) + (or (keyword? (first x)) + (integer? (first x))))) + +(defn > + (vec (for [[op & entity] (:entities report)] + (into [op] (for [field entity] + (if (lookup-ref? field) + (first ( + report + (assoc-in [:tempids tempid] upserted-eid) + (assoc-in [:entities] es))))) + +(defn- transact-entity [report entity] + (update-in report [:entities] conj entity)) + +(defn datom. + (doseq [[e a v tx added :as datom] (:tx-data report)] + + (when added + ;; Check for violated :db/unique constraint between datom and existing store. + (when (ds/unique? schema a) + (when-let [found (first (tx-data [db report] + {:pre [(db/db? db) (report? report)]} + (go-pair + (let [initial-report report + {tx :tx} report + schema (db/schema db)] + (loop [report initial-report + es (:entities initial-report)] + (let [[[op e a v :as entity] & entities] es] + (cond + (nil? entity) + report + + (= op :db/add) + (if (ds/multival? schema a) + (if (empty? ( report + (transact-report (datom e a (.-v old-datom) tx false)) + (transact-report (datom e a v tx true))) + entities)) + (recur (transact-report report (datom e a v tx true)) entities))) + + (= op :db/retract) + (if (first (> + report + (preprocess db) + + (tx-data db) + (schema-fragment datoms)] + (assoc-in report [:added-attributes] schema-fragment))) + +;; TODO: expose this in a more appropriate way. +(defn > + (map->TxReport + {:db-before db + :db-after db + ;; This mimics DataScript. It's convenient to be able to extract the + ;; transaction ID and transaction timestamp directly from the report; Datomic + ;; makes this surprisingly difficult: one needs a :db.part/tx temporary and an + ;; explicit upsert of that temporary. + :tx (db/current-tx db) + :txInstant (db/now db) + :entities tx-data + :tx-data [] + :tempids {} + :added-parts {} + :added-idents {} + :added-attributes {} + }) + + ( + db + + (db/ report + (assoc-in [:db-after] db-after))))) + +(defn- !]]]) + #?@(:cljs [[datomish.pair-chan] + [cljs.core.async :as a :refer [!]]]))) + +(defn + (sqlite/> (s/all-rows (:sqlite-connection db) ["SELECT e, a, v, tx FROM datoms WHERE tx > ?" tx]) @@ -51,7 +51,7 @@ (defn- > (s/all-rows (:sqlite-connection db) ["SELECT a, v FROM datoms WHERE e = ?" eid]) @@ -60,7 +60,7 @@ (reduce conj {}))))) (defn- > (s/all-rows (:sqlite-connection db) ["SELECT e, a, v, tx, added FROM transactions WHERE tx > ? ORDER BY tx ASC, e, a, v, added" tx]) @@ -79,80 +79,76 @@ ;; TODO: use reverse refs! (def test-schema - [{:db/id (dm/id-literal :test -1) + [{:db/id (d/id-literal :test -1) :db/ident :x :db/unique :db.unique/identity :db/valueType :db.type/integer} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -1)} - {:db/id (dm/id-literal :test -2) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -1)} + {:db/id (d/id-literal :test -2) :db/ident :name :db/unique :db.unique/identity :db/valueType :db.type/string} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -2)} - {:db/id (dm/id-literal :test -3) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -2)} + {:db/id (d/id-literal :test -3) :db/ident :y :db/cardinality :db.cardinality/many :db/valueType :db.type/integer} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -3)} - {:db/id (dm/id-literal :test -5) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -3)} + {:db/id (d/id-literal :test -5) :db/ident :aka :db/cardinality :db.cardinality/many :db/valueType :db.type/string} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -5)} - {:db/id (dm/id-literal :test -6) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -5)} + {:db/id (d/id-literal :test -6) :db/ident :age :db/valueType :db.type/integer} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -6)} - {:db/id (dm/id-literal :test -7) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -6)} + {:db/id (d/id-literal :test -7) :db/ident :email :db/unique :db.unique/identity :db/valueType :db.type/string} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -7)} - {:db/id (dm/id-literal :test -8) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -7)} + {:db/id (d/id-literal :test -8) :db/ident :spouse :db/unique :db.unique/value :db/valueType :db.type/string} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -8)} - {:db/id (dm/id-literal :test -9) + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -8)} + {:db/id (d/id-literal :test -9) :db/ident :friends :db/cardinality :db.cardinality/many :db/valueType :db.type/ref} - {:db/id :db.part/db :db.install/attribute (dm/id-literal :test -9)} + {:db/id :db.part/db :db.install/attribute (d/id-literal :test -9)} ]) (deftest-async test-add-one (with-tempfile [t (tempfile)] - (let [c (