diff --git a/src/common/datomish/api.cljc b/src/common/datomish/api.cljc index 6ceb93e4..567335e0 100644 --- a/src/common/datomish/api.cljc +++ b/src/common/datomish/api.cljc @@ -32,6 +32,8 @@ (def id-literal db/id-literal) +(def lookup-ref db/lookup-ref) + (def db transact/db) (def entid db/entid) diff --git a/src/common/datomish/db.cljc b/src/common/datomish/db.cljc index 5fd28115..c41381a8 100644 --- a/src/common/datomish/db.cljc +++ b/src/common/datomish/db.cljc @@ -63,6 +63,23 @@ (defn id-literal? [x] (instance? TempId x)) +(defrecord LookupRef [a v]) + +(defn lookup-ref + [a v] + (if (and + (or (keyword? a) + (integer? a)) + v) + (->LookupRef a v) + (raise (str "Lookup-ref with bad attribute " a " or value " v + {:error :transact/bad-lookup-ref, :a a, :v v})))) + +(defn lookup-ref? [x] + "Return `x` if `x` is like [:attr value], nil otherwise." + (when (instance? LookupRef x) + x)) + (defprotocol IClock (now [clock] @@ -105,6 +122,13 @@ [db a v] "Search for a single matching datom using the AVET index.") + (queries [eas tx] +(defn- retractAttributes->queries [oeas tx] (let [where-part "(e = ? AND a = ?)" @@ -183,21 +205,23 @@ (fn [chunk] (cons (apply str - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag) - SELECT e, a, v, ?, 0, value_type_tag, v, value_type_tag - FROM datoms - WHERE " + "INSERT INTO temp.tx_lookup_after (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag, + rid, e, a, v, tx, value_type_tag) + SELECT e, a, v, ?, 0, value_type_tag, v, value_type_tag, + rowid, e, a, v, ?, value_type_tag + FROM datoms + WHERE " (repeater (count chunk))) (cons tx - (mapcat (fn [[_ e a]] - [e a]) - chunk)))) - (partition-all (quot (dec max-sql-vars) 2) eas)))) + (cons + tx + (mapcat (fn [[_ e a]] + [e a]) + chunk))))) + (partition-all (quot (- max-sql-vars 2) 2) oeas)))) -;; TODO: make this not do the tx_lookup. We could achieve this by having additional special values -;; of added0, or by separating the tx_lookup table into before and after tables. -(defn- retractEntities->queries [es tx] +(defn- retractEntities->queries [oes tx] (let [ref-tag (sqlite-schema/->tag :db.type/ref) ;; TODO: include index_vaet flag here, so we can use that index to speed up the deletion. @@ -209,27 +233,31 @@ (fn [chunk] (cons (apply str - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag) - SELECT e, a, v, ?, 0, value_type_tag, v, value_type_tag - FROM datoms - WHERE " + "INSERT INTO temp.tx_lookup_after (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag, + rid, e, a, v, tx, value_type_tag) + SELECT e, a, v, ?, 0, value_type_tag, v, value_type_tag, + rowid, e, a, v, ?, value_type_tag + FROM datoms + WHERE " (repeater (count chunk))) (cons tx - (mapcat (fn [[_ e]] - [e e]) - chunk)))) - (partition-all (quot (dec max-sql-vars) 2) es)))) + (cons + tx + (mapcat (fn [[_ e]] + [e e]) + chunk))))) + (partition-all (quot (- max-sql-vars 2) 2) oes)))) (defn- retractions->queries [retractions tx fulltext? ->SQLite] (let [f-q "WITH vv AS (SELECT rowid FROM fulltext_values WHERE text = ?) - INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag) + INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag) VALUES (?, ?, (SELECT rowid FROM vv), ?, 0, ?, (SELECT rowid FROM vv), ?)" non-f-q - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag) + "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, sv, svalue_type_tag) VALUES (?, ?, ?, ?, 0, ?, ?, ?)"] (map (fn [[_ e a v]] @@ -242,7 +270,7 @@ retractions))) (defn- non-fts-many->queries [ops tx ->SQLite indexing? ref? unique?] - (let [q "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " + (let [q "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " values-part ;; e0, a0, v0, tx0, added0, value_type_tag0 @@ -290,7 +318,7 @@ [(cons (apply str - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " + "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " (first-repeater (count chunk))) (mapcat (fn [[_ e a v]] (let [[v tag] (->SQLite a v)] @@ -304,7 +332,7 @@ (cons (apply str - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0) VALUES " + "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0) VALUES " (second-repeater (count chunk))) (mapcat (fn [[_ e a v]] (let [[v tag] (->SQLite a v)] @@ -341,10 +369,10 @@ [["INSERT INTO fulltext_values_view (text, searchid) VALUES (?, ?)" v searchid] - ;; Second query: tx_lookup. + ;; Second query: lookup. [(str "WITH vv(rowid) AS (SELECT rowid FROM fulltext_values WHERE searchid = ?) " - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " + "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " "(?, ?, (SELECT rowid FROM vv), ?, 1, ?, ?, ?, 1, ?, (SELECT rowid FROM vv), ?)") searchid e a tx tag @@ -365,10 +393,10 @@ [["INSERT INTO fulltext_values_view (text, searchid) VALUES (?, ?)" v searchid] - ;; Second and third queries: tx_lookup. + ;; Second and third queries: lookup. [(str "WITH vv(rowid) AS (SELECT rowid FROM fulltext_values WHERE searchid = ?) " - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " + "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0, index_avet0, index_vaet0, index_fulltext0, unique_value0, sv, svalue_type_tag) VALUES " "(?, ?, (SELECT rowid FROM vv), ?, 1, ?, ?, ?, 1, ?, (SELECT rowid FROM vv), ?)") searchid e a tx tag @@ -378,7 +406,7 @@ tag] [(str - "INSERT INTO tx_lookup (e0, a0, v0, tx0, added0, value_type_tag0) VALUES " + "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0) VALUES " "(?, ?, (SELECT rowid FROM fulltext_values WHERE searchid = ?), ?, 0, ?)") e a searchid tx tag]])) ops @@ -390,33 +418,43 @@ (try (doseq [q queries] ( searchid. + av->searchid + (into {} (map vector avs (range))) + + ;; Each query takes 4 variables per item. So we partition into max-sql-vars / 4. + qs + (map + (fn [chunk] + (cons + ;; Query string. + (apply str "WITH t(searchid, a, v, value_type_tag) AS (VALUES " + (apply str (repeater (count chunk))) ;; TODO: join? + ") SELECT t.searchid, d.e + FROM t, datoms AS d + WHERE d.index_avet IS NOT 0 AND d.a = t.a AND d.value_type_tag = t.value_type_tag AND d.v = t.v") + + ;; Bindings. + (mapcat (fn [[[a v] searchid]] + (let [a (entid db a) + [v tag] (ds/->SQLite schema a v)] + [searchid a v tag])) + chunk))) + + (partition-all (quot max-sql-vars 4) av->searchid)) + + ;; Map searchid -> e. There's a generic reduce that takes [pair-chan] lurking in here. + searchid->e + (loop [coll (transient {}) + qs qs] + (let [[q & qs] qs] + (if q + (let [rs (e) av->searchid)))) + (= [db tx] + (go-pair + (->> + (s/all-rows (:sqlite-connection db) ["SELECT e, a, v, tx FROM datoms WHERE tx >= ?" tx]) + (> diff --git a/src/common/datomish/transact.cljc b/src/common/datomish/transact.cljc index 70e63fac..f4ef7e10 100644 --- a/src/common/datomish/transact.cljc +++ b/src/common/datomish/transact.cljc @@ -61,7 +61,7 @@ db-after ;; The DB after the transaction. tx ;; The tx ID represented by the transaction in this report; refer :db/tx. txInstant ;; The timestamp instant when the the transaction was processed/committed in this report; refer :db/txInstant. - entities ;; The set of entities (like [:db/add e a v tx]) processed. + entities ;; The set of entities (like [:db/add e a v]) processed. tx-data ;; The set of datoms applied to the database, like (Datom. e a v tx added). tempids ;; The map from id-literal -> numeric entid. part-map ;; Map {:db.part/user {:start 0x10000 :idx 0x10000}, ...}. @@ -118,7 +118,7 @@ true entity)) -(defn maybe-ident->entid [db [op e a v tx :as orig]] +(defn maybe-ident->entid [db [op e a v :as orig]] ;; We have to handle all ops, including those when a or v are not defined. (let [e (db/entid db e) a (db/entid db a) @@ -127,8 +127,8 @@ (db/entid db v))] (when (and a (not (integer? a))) (raise "Unknown attribute " a - {:form orig :attribute a})) - [op e a v tx])) + {:form orig :attribute a :entity orig})) + [op e a v])) (defrecord Transaction [db tempids entities]) @@ -250,49 +250,43 @@ ;; Extract the current txInstant for the report. (->> (update-txInstant db*))))) -(defn- lookup-ref? [x] - "Return `x` if `x` is like [:attr value], false otherwise." - (and (sequential? x) - (= (count x) 2) - (or (keyword? (first x)) - (integer? (first x))) - x)) - (defn entities containing lookup-ref, like {[[:a :v] [[[:a :v] :b :w] ...]] ...}. + groups (group-by (partial keep db/lookup-ref?) (:entities report)) + ;; Entities with no lookup-ref are grouped under the key (lazy-seq). + entities (get groups (lazy-seq)) ;; No lookup-refs? Pass through. + to-resolve (dissoc groups (lazy-seq)) ;; The ones with lookup-refs. + ;; List [[:a :v] ...] to lookup. + avs (set (map (juxt :a :v) (apply concat (keys to-resolve)))) + ->av (fn [r] ;; Conditional (juxt :a :v) that passes through nil. + (when r [(:a r) (:v r)]))] (go-pair - (if (empty? entities) - report - (assoc-in - report [:entities] - ;; We can't use `for` because go-pair doesn't traverse function boundaries. - ;; Apologies for the tortured nested loop. - (loop [[op & entity] (first entities) - next (rest entities) - acc []] - (if (nil? op) - acc - (recur (first next) - (rest next) - (conj acc - (loop [field (first entity) - rem (rest entity) - acc [op]] - (if (nil? field) - acc - (recur (first rem) - (rest rem) - (conj acc - (if-let [[a v] (lookup-ref? field)] - (or - ;; The lookup might fail! If so, throw. - (:e (e (av (db/lookup-ref? field))] + (if-not (unique-identity? (db/entid db a)) + (raise "Lookup-ref found with non-unique-identity attribute " a " and value " v + {:error :transact/lookup-ref-with-non-unique-identity-attribute + :a a + :v v}) + (or + (get av->e [a v]) + (raise "No entity found for lookup-ref with attribute " a " and value " v + {:error :transact/lookup-ref-not-found + :a a + :v v}))) + field)) + resolve (fn [entity] + (mapv resolve1 entity))] + (assoc + report + :entities + (concat + entities + (map resolve (apply concat (vals to-resolve))))))))) (declare = schema in)) - expected))))) + (is (= (map #(dissoc %1 :db/id) (datomish.simple-schema/simple-schema->schema in)) + expected))))) +(deftest-db test-lookup-refs conn + (let [{tx0 :tx} (= (d/db conn) tx1)))))) + + (testing "Looks up value refs" + (let [{tx :tx} (= (d/db conn) tx)))))) + + (testing "Looks up entity refs in maps" + (let [{tx :tx} (= (d/db conn) tx)))))) + + (testing "Looks up value refs in maps" + (let [{tx :tx} (= (d/db conn) tx)))))) + + (testing "Looks up value refs in sequences in maps" + (let [{tx :tx} (= (d/db conn) tx)))))) + + (testing "Looks up refs when there are more than 999 refs (all present)" + (let + [bound (* 999 2) + make-add #(vector :db/add (+ 1000 %) :name (str "Ivan-" %)) + make-ref #(-> {:db/id (d/lookup-ref :name (str "Ivan-" %)) :email (str "Ivan-" % "@" %)}) + {tx-data1 :tx-data} (