From d92016166a39647e7b96f037619c27defcde1276 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 31 Aug 2016 16:04:46 -0700 Subject: [PATCH] Cache partition map and update materialized partition view once. Fixes #47. This caches a partition map per DB, which is helpful because it exposes what the point in time DB partition state is, but is unhelpful because the partition state can advance underneath the DB cache. This is generally true of the approach -- this can happen to the ident/entid maps, and the datoms themselves -- so we'll roll with it for now. This reduces the number of SQL UPDATE operations from linear in the number of id-literals used to constant in the number of known partitions. --- src/datomish/db.cljc | 63 ++++++++++++++++----------- src/datomish/db_factory.cljc | 65 ++++++++++++++++++---------- src/datomish/sqlite_schema.cljc | 2 +- src/datomish/transact.cljc | 60 ++++++++++++++++--------- src/datomish/transact/bootstrap.cljc | 7 +++ 5 files changed, 127 insertions(+), 70 deletions(-) diff --git a/src/datomish/db.cljc b/src/datomish/db.cljc index 6608a972..4e1b718f 100644 --- a/src/datomish/db.cljc +++ b/src/datomish/db.cljc @@ -88,6 +88,10 @@ [db eid] "Returns the keyword associated with an id, or the key itself if passed.") + (part-map + [db] + "Return the partition map of this database, like {:db.part/user {:start 0x100 :idx 0x101}, ...}.") + (in-transaction! [db chan-fn] "Evaluate the given pair-chan `chan-fn` in an exclusive transaction. If it returns non-nil, @@ -113,10 +117,9 @@ [db fragment merge] "Apply added schema fragment to the store, using `merge` as a `merge-with` function.") - (SQLite part) idx])) + part-map)] + ;; TODO: chunk into 999/2 sections, for safety. + (when-not (empty? pairs) + (DB {:sqlite-connection sqlite-connection :ident-map ident-map + :part-map parts :symbolic-schema schema :schema entid-schema - ;; TODO :parts }))) ;; TODO: factor this into the overall design. diff --git a/src/datomish/db_factory.cljc b/src/datomish/db_factory.cljc index 2c8b7131..8e875cd5 100644 --- a/src/datomish/db_factory.cljc +++ b/src/datomish/db_factory.cljc @@ -34,6 +34,17 @@ (s/all-rows sqlite-connection)))] (into {} (map (fn [row] [(sqlite-schema/<-SQLite :db.type/keyword (:ident row)) (:entid row)])) rows)))) +(defn {:start integer :idx integer}, like {:db.part/user {start: 0x100 idx: 0x101}}." + + (go-pair + (let [rows (> + {:select [:part :start :idx] :from [:parts]} + (s/format) + (s/all-rows sqlite-connection)))] + (into {} (map (fn [row] [(sqlite-schema/<-SQLite :db.type/keyword (:part row)) (select-keys row [:start :idx])])) rows)))) + (defn (map (keyword attribute -> keyword value)), like @@ -72,7 +83,7 @@ (when-not (= sqlite-schema/current-version ( db - ;; We use vector (fn [[part {:keys [start idx]}]] + (println "part->vector" part start idx) + [(sqlite-schema/->SQLite part) start idx])] + ;; TODO: allow inserting new parts. + ;; TODO: think more carefully about allocating new parts and bitmasking part ranges. + ;; TODO: install these using bootstrap assertions. It's tricky because the part ranges are implicit. + ;; TODO: chunk into 999/3 sections, for safety. + (vector bootstrap/parts))))) + + (-> db + ;; We use numeric entid. + part-map ;; Map {:db.part/user {:start 0x10000 :idx 0x10000}, ...}. added-parts ;; The set of parts added during the transaction via :db.part/db :db.install/part. added-idents ;; The map of idents -> entid added during the transaction, via e :db/ident ident. added-attributes ;; The map of schema attributes (ident -> schema fragment) added during the transaction, via :db.part/db :db.install/attribute. @@ -72,11 +73,19 @@ (defn- report? [x] (and (instance? TxReport x))) -(defonce -eid (atom (- 0x200 1))) - -;; TODO: better here. -(defn- next-eid [db] - (swap! -eid inc)) +(defn- -next-eid! [part-map-atom tempid] + "Advance {:db.part/user {:start 0x10 :idx 0x11}, ...} to {:db.part/user {:start 0x10 :idx 0x12}, ...} and return 0x12." + {:pre [(id-literal? tempid)]} + (let [part (:part tempid) + next (fn [part-map] + (let [idx (get-in part-map [part :idx])] + (when-not idx + (raise "Cannot allocate entid for id-literal " tempid " because part " part " is not known" + {:error :db/bad-part + :parts (sorted-set (keys part-map)) + :part part})) + (update-in part-map [part :idx] inc)))] + (get-in (swap! part-map-atom next) [part :idx]))) (defn- allocate-eid [report id-literal eid] @@ -327,22 +336,22 @@ allocated-eid (get-in report [:tempids e])] (if (and upserted-eid allocated-eid (not= upserted-eid allocated-eid)) (> + (let [part-map-atom + (atom (db/part-map db)) + + tx + (-next-eid! part-map-atom (id-literal :db.part/tx)) + + report (->> (map->TxReport - {:db-before db - :db-after db + {:db-before db + :db-after db ;; This mimics DataScript. It's convenient to be able to extract the ;; transaction ID and transaction timestamp directly from the report; Datomic ;; makes this surprisingly difficult: one needs a :db.part/tx temporary and an ;; explicit upsert of that temporary. - :tx ( db + (db/> (p :apply-db-part-changes)) + (db/> (p :apply-db-ident-assertions)) diff --git a/src/datomish/transact/bootstrap.cljc b/src/datomish/transact/bootstrap.cljc index 17c90be2..9a7f2e9a 100644 --- a/src/datomish/transact/bootstrap.cljc +++ b/src/datomish/transact/bootstrap.cljc @@ -76,9 +76,16 @@ :db/doc 35 }) +(def parts + {:db.part/db {:start 0 :idx (inc (apply max (vals idents)))} + :db.part/user {:start 0x10000 :idx 0x10000} + :db.part/tx {:start 0x10000000 :idx 0x10000000} + }) + (defn tx-data [] (concat (map (fn [[ident entid]] [:db/add entid :db/ident ident]) idents) + ;; TODO: install partitions as well, like (map (fn [[ident entid]] [:db/add :db.part/db :db.install/partition ident])). (map (fn [[ident attrs]] (assoc attrs :db/id ident)) symbolic-schema) (map (fn [[ident attrs]] [:db/add :db.part/db :db.install/attribute (get idents ident)]) symbolic-schema) ;; TODO: fail if nil. ))