Tag values with value type tags in SQLite.

src/datomish/db.cljc

@@ -92,12 +92,14 @@
     commit the transaction; otherwise, rollback the transaction.  Returns a pair-chan resolving to
     the pair-chan returned by `chan-fn`.")
 
-  (<eavt
+  (<ea [db e a]
-    [db pattern]
     "Search for datoms using the EAVT index.")
 
-  (<avet
+  (<eav [db e a v]
-    [db pattern]
+    "Search for datoms using the EAVT index.")
+
+  (<av
+    [db a v]
     "Search for datoms using the AVET index.")
 
   (<apply-datoms
@@ -195,30 +197,41 @@
       (:sqlite-connection db) chan-fn))
 
   ;; TODO: use q for searching?  Have q use this for searching for a single pattern?
-  (<eavt [db pattern]
+  (<ea [db e a]
-    (let [[e a v] pattern
+    (go-pair
-          v       (and v (ds/->SQLite schema a v))] ;; We assume e and a are always given.
+      (->>
+        {:select [:e :a :v :tx [1 :added]]
+         :from   [:all_datoms]
+         :where  [:and [:= :e e] [:= :a a]]}
+        (s/format) ;; TODO: format these statements only once.
+
+        (s/all-rows (:sqlite-connection db))
+        (<?)
+
+        (mapv (partial row->Datom (.-schema db))))))
+
+  (<eav [db e a v]
+    (let [[v tag] (ds/->SQLite schema a v)]
       (go-pair
         (->>
           {:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns.
            :from   [:all_datoms]
-           :where  (cons :and (map #(vector := %1 %2) [:e :a :v] (take-while (comp not nil?) [e a v])))} ;; Must drop nils.
+           :where  [:and [:= :e e] [:= :a a] [:= :value_type_tag tag] [:= :v v]]}
-          (s/format)
+          (s/format) ;; TODO: format these statements only once.
 
           (s/all-rows (:sqlite-connection db))
           (<?)
 
           (mapv (partial row->Datom (.-schema db))))))) ;; TODO: understand why (schema db) fails.
 
-  (<avet [db pattern]
+  (<av [db a v]
-    (let [[a v] pattern
+    (let [[v tag] (ds/->SQLite schema a v)]
-          v     (ds/->SQLite schema a v)]
       (go-pair
         (->>
           {:select [:e :a :v :tx [1 :added]] ;; TODO: generalize columns.
            :from   [:all_datoms]
-           :where  [:and [:= :a a] [:= :v v] [:= :index_avet 1]]}
+           :where  [:and [:= :index_avet 1] [:= :a a] [:= :value_type_tag tag] [:= :v v]]}
-          (s/format)
+          (s/format) ;; TODO: format these statements only once.
 
           (s/all-rows (:sqlite-connection db))
           (<?)
@@ -232,18 +245,19 @@
         ;; TODO: batch insert, batch delete.
         (doseq [datom datoms]
           (let [[e a v tx added] datom
-                v                (ds/->SQLite schema a v)
+                [v tag]          (ds/->SQLite schema a v)
                 fulltext?        (ds/fulltext? schema a)]
             ;; Append to transaction log.
             (<? (exec
-                  ["INSERT INTO transactions VALUES (?, ?, ?, ?, ?)" e a v tx (if added 1 0)]))
+                  ["INSERT INTO transactions VALUES (?, ?, ?, ?, ?, ?)" e a v tx (if added 1 0) tag]))
             ;; Update materialized datom view.
             (if (.-added datom)
               (let [v (if fulltext?
                         (<? (<insert-fulltext-value db v))
                         v)]
                 (<? (exec
-                      ["INSERT INTO datoms VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" e a v tx
+                      ["INSERT INTO datoms VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" e a v tx
+                       tag ;; value_type_tag
                        (ds/indexing? schema a) ;; index_avet
                        (ds/ref? schema a) ;; index_vaet
                        fulltext? ;; index_fulltext
@@ -253,9 +267,9 @@
               (if fulltext?
                 (<? (exec
                       ;; TODO: in the future, purge fulltext values from the fulltext_datoms table.
-                      ["DELETE FROM datoms WHERE (e = ? AND a = ? AND v IN (SELECT rowid FROM fulltext_values WHERE text = ?))" e a v]))
+                      ["DELETE FROM datoms WHERE (e = ? AND a = ? AND value_type_tag = ? AND v IN (SELECT rowid FROM fulltext_values WHERE text = ?))" e a tag v]))
                 (<? (exec
-                      ["DELETE FROM datoms WHERE (e = ? AND a = ? AND v = ?)" e a v])))))))
+                      ["DELETE FROM datoms WHERE (e = ? AND a = ? AND value_type_tag = ? AND v = ?)" e a tag v])))))))
       db))
 
   (<advance-tx [db]
@@ -270,7 +284,7 @@
 
   (<apply-db-ident-assertions [db added-idents merge]
     (go-pair
-      (let [exec     (partial s/execute! (:sqlite-connection db))]
+      (let [exec (partial s/execute! (:sqlite-connection db))]
         ;; TODO: batch insert.
         (doseq [[ident entid] added-idents]
           (<? (exec
@@ -282,7 +296,7 @@
 
   (<apply-db-install-assertions [db fragment merge]
     (go-pair
-      (let [exec     (partial s/execute! (:sqlite-connection db))]
+      (let [exec (partial s/execute! (:sqlite-connection db))]
         ;; TODO: batch insert.
         (doseq [[ident attr-map] fragment]
           (doseq [[attr value] attr-map]

src/datomish/schema.cljc

@@ -101,8 +101,8 @@
    :db.type/keyword { :valid? keyword? }
    :db.type/string  { :valid? string? }
    :db.type/boolean { :valid? #?(:clj #(instance? Boolean %) :cljs #(= js/Boolean (type %))) }
-   :db.type/integer { :valid? integer? }
+   :db.type/long    { :valid? integer? }
-   :db.type/real    { :valid? #?(:clj float? :cljs number?) }
+   :db.type/double  { :valid? #?(:clj float? :cljs number?) }
    })
 
 (defn #?@(:clj  [^Boolean ensure-valid-value]
@@ -125,7 +125,7 @@
     (if-let [valueType (get-in schema [attr :db/valueType])]
       (if-let [valid? (get-in value-type-map [valueType :valid?])]
         (if (valid? value)
-          (sqlite-schema/->SQLite value)
+          [(sqlite-schema/->SQLite value) (sqlite-schema/->tag valueType)]
           (raise "Invalid value for attribute " attr ", expected " valueType " but got " value
                  {:error :schema/valueType, :attribute attr, :value value}))
         (raise "Unknown valueType for attribute " attr ", expected one of " (sorted-set (keys value-type-map))

src/datomish/sqlite_schema.cljc

@@ -8,7 +8,7 @@
       [datomish.pair-chan :refer [go-pair <?]]
       [cljs.core.async.macros :refer [go]]))
   (:require
-   [datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise-str cond-let]]
+   [datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise raise-str cond-let]]
    [datomish.sqlite :as s]
    #?@(:clj [[datomish.pair-chan :refer [go-pair <?]]
              [clojure.core.async :refer [go <! >!]]])
@@ -17,35 +17,54 @@
 
 (def current-version 1)
 
+;; Datomish rows are tagged with a numeric representation of :db/valueType:
+;; The tag is used to limit queries, and therefore is placed carefully in the relevant indices to
+;; allow searching numeric longs and doubles quickly.  The tag is also used to convert SQLite values
+;; to the correct Datomish value type on query egress.
+(def value-type-tag-map
+  {:db.type/ref     0
+   :db.type/boolean 1
+   :db.type/instant 4
+   :db.type/long    5 ;; SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
+   :db.type/double  5 ;; SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
+   :db.type/string  10
+   :db.type/uuid    11
+   :db.type/uri     12
+   :db.type/keyword 13})
+
 (def v1-statements
   ["CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL,
+                         value_type_tag SMALLINT NOT NULL,
                          index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0,
                          index_fulltext TINYINT NOT NULL DEFAULT 0,
                          unique_value TINYINT NOT NULL DEFAULT 0, unique_identity TINYINT NOT NULL DEFAULT 0)"
-   "CREATE INDEX idx_datoms_eavt ON datoms (e, a, v)"
+   "CREATE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"
-   "CREATE INDEX idx_datoms_aevt ON datoms (a, e, v)"
+   "CREATE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"
 
    ;; Opt-in index: only if a has :db/index true.
-   "CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, v, e) WHERE index_avet IS NOT 0"
+   "CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"
 
-   ;; Opt-in index: only if a has :db/valueType :db.type/ref.
+   ;; Opt-in index: only if a has :db/valueType :db.type/ref.  No need for tag here since all
+   ;; indexed elements are refs.
    "CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0"
 
    ;; Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string,
    ;; which is not :db/valueType :db.type/ref.  That is, index_vaet and index_fulltext are mutually
    ;; exclusive.
-   "CREATE INDEX idx_datoms_fulltext ON datoms (v, a, e) WHERE index_fulltext IS NOT 0"
+   "CREATE INDEX idx_datoms_fulltext ON datoms (value_type_tag, v, a, e) WHERE index_fulltext IS NOT 0"
 
    ;; TODO: possibly remove this index.  :db.unique/value should be asserted by the transactor in
    ;; all cases, but the index may speed up some of SQLite's query planning.  For now, it services
-   ;; to validate the transactor implementation.
+   ;; to validate the transactor implementation.  Note that tag is needed here, since we could have
-   "CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (v) WHERE unique_value IS NOT 0"
+   ;; a keyword (stored as ":foo") that overlaps a string value ":foo".
+   "CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (value_type_tag, v) WHERE unique_value IS NOT 0"
    ;; TODO: possibly remove this index.  :db.unique/identity should be asserted by the transactor in
    ;; all cases, but the index may speed up some of SQLite's query planning.  For now, it serves to
-   ;; validate the transactor implementation.
+   ;; validate the transactor implementation.  Note that tag is needed here to differentiate, e.g.,
-   "CREATE UNIQUE INDEX idx_datoms_unique_identity ON datoms (a, v) WHERE unique_identity IS NOT 0"
+   ;; keywords and strings.
+   "CREATE UNIQUE INDEX idx_datoms_unique_identity ON datoms (a, value_type_tag, v) WHERE unique_identity IS NOT 0"
 
-   "CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1)"
+   "CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1, value_type_tag SMALLINT NOT NULL)"
    "CREATE INDEX idx_transactions_tx ON transactions (tx)"
 
    ;; Fulltext indexing.
@@ -61,21 +80,22 @@
 
    ;; A view transparently interpolating fulltext indexed values into the datom structure.
    "CREATE VIEW fulltext_datoms AS
-     SELECT e, a, fulltext_values.text AS v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
+     SELECT e, a, fulltext_values.text AS v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
        FROM datoms, fulltext_values
        WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid"
 
    ;; A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure.
    "CREATE VIEW all_datoms AS
-     SELECT e, a, v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
+     SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
        FROM datoms
        WHERE index_fulltext IS 0
      UNION ALL
-     SELECT e, a, v, tx, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
+     SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value, unique_identity
        FROM fulltext_datoms"
 
    ;; Materialized views of the schema.
    "CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"
+   ;; TODO: allow arbitrary schema values (true/false) and tag the resulting values.
    "CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (ident) REFERENCES idents (ident))"
    "CREATE INDEX idx_schema_unique ON schema (ident, attr, value)"
    ])
@@ -142,6 +162,7 @@
 
        Double
        (->SQLite [x] x)]
+
       :cljs
       [string
        (->SQLite [x] x)
@@ -162,5 +183,11 @@
     :db.type/keyword (keyword (subs value 1))
     :db.type/string  value
     :db.type/boolean (not= value 0)
-    :db.type/integer value
+    :db.type/long    value
-    :db.type/real    value))
+    :db.type/double  value))
+
+(defn ->tag [valueType]
+  (or
+    (valueType value-type-tag-map)
+    (raise "Unknown valueType " valueType ", expected one of " (sorted-set (keys value-type-tag-map))
+           {:error :SQLite/tag, :valueType valueType})))

src/datomish/transact.cljc

@@ -219,7 +219,7 @@
       (vec (for [[op & entity] (:entities report)]
              (into [op] (for [field entity]
                           (if (lookup-ref? field)
-                            (first (<? (db/<eavt db field))) ;; TODO improve this -- this should be avet, shouldn't it?
+                            (first (<? (apply db/<av db field)))
                             field)))))
       (assoc-in report [:entities])))) ;; TODO: meta.
 
@@ -289,7 +289,7 @@
             (and (id-literal? e)
                  (ds/unique-identity? (db/schema db) a)
                  (not-any? id-literal? [a v]))
-            (let [upserted-eid (:e (first (<? (db/<avet db [a v]))))
+            (let [upserted-eid (:e (first (<? (db/<av db a v))))
                   allocated-eid (get-in report [:tempids e])]
               (if (and upserted-eid allocated-eid (not= upserted-eid allocated-eid))
                 (<? (<retry-with-tempid db initial-report initial-entities e upserted-eid)) ;; TODO: not initial report, just the sorted entities here.
@@ -351,7 +351,7 @@
         (when added
           ;; Check for violated :db/unique constraint between datom and existing store.
           (when (ds/unique? schema a)
-            (when-let [found (first (<? (db/<avet db [a v])))]
+            (when-let [found (first (<? (db/<av db a v)))]
               (raise "Cannot add " datom " because of unique constraint: " found
                      {:error :transact/unique
                       :attribute a ;; TODO: map attribute back to ident.
@@ -401,10 +401,10 @@
 
             (= op :db/add)
             (if (ds/multival? schema a)
-              (if (empty? (<? (db/<eavt db [e a v])))
+              (if (empty? (<? (db/<eav db e a v)))
                 (recur (transact-report report (datom e a v tx true)) entities)
                 (recur report entities))
-              (if-let [^Datom old-datom (first (<? (db/<eavt db [e a])))]
+              (if-let [^Datom old-datom (first (<? (db/<ea db e a)))]
                 (if  (= (.-v old-datom) v)
                   (recur report entities)
                   (recur (-> report
@@ -414,7 +414,7 @@
                 (recur (transact-report report (datom e a v tx true)) entities)))
 
             (= op :db/retract)
-            (if (first (<? (db/<eavt db [e a v])))
+            (if (first (<? (db/<eav db e a v)))
               (recur (transact-report report (datom e a v tx false)) entities)
               (recur report entities))
 

src/datomish/transact/bootstrap.cljc

@@ -17,7 +17,7 @@
    ;; TODO: support user-specified functions in the future.
    ;; :db.install/function {:db/valueType :db.type/ref
    ;;                       :db/cardinality :db.cardinality/many}
-   :db/txInstant         {:db/valueType   :db.type/integer
+   :db/txInstant         {:db/valueType   :db.type/long
                           :db/cardinality :db.cardinality/one
                           } ;; :db/index       true} TODO: Handle this using SQLite protocol.
    :db/valueType         {:db/valueType   :db.type/ref
@@ -61,15 +61,16 @@
    :db.alter/attribute   22
    :db.type/ref          23
    :db.type/keyword      24
-   :db.type/integer      25 ;; TODO: :db.type/long, to match Datomic?
+   :db.type/long         25
-   :db.type/string       26
+   :db.type/double       26
-   :db.type/boolean      27
+   :db.type/string       27
-   :db.type/instant      28
+   :db.type/boolean      28
-   :db.type/bytes        29
+   :db.type/instant      29
-   :db.cardinality/one   30
+   :db.type/bytes        30
-   :db.cardinality/many  31
+   :db.cardinality/one   31
-   :db.unique/value      32
+   :db.cardinality/many  32
-   :db.unique/identity   33})
+   :db.unique/value      33
+   :db.unique/identity   34})
 
 (defn tx-data []
   (concat

test/datomish/db_test.cljc

@@ -78,7 +78,7 @@
   [{:db/id        (d/id-literal :test -1)
     :db/ident     :x
     :db/unique    :db.unique/identity
-    :db/valueType :db.type/integer}
+    :db/valueType :db.type/long}
    {:db/id :db.part/db :db.install/attribute (d/id-literal :test -1)}
    {:db/id        (d/id-literal :test -2)
     :db/ident     :name
@@ -88,7 +88,7 @@
    {:db/id          (d/id-literal :test -3)
     :db/ident       :y
     :db/cardinality :db.cardinality/many
-    :db/valueType   :db.type/integer}
+    :db/valueType   :db.type/long}
    {:db/id :db.part/db :db.install/attribute (d/id-literal :test -3)}
    {:db/id          (d/id-literal :test -5)
     :db/ident       :aka
@@ -97,7 +97,7 @@
    {:db/id :db.part/db :db.install/attribute (d/id-literal :test -5)}
    {:db/id        (d/id-literal :test -6)
     :db/ident     :age
-    :db/valueType :db.type/integer}
+    :db/valueType :db.type/long}
    {:db/id :db.part/db :db.install/attribute (d/id-literal :test -6)}
    {:db/id        (d/id-literal :test -7)
     :db/ident     :email

test/datomish/query_test.cljc

@@ -32,7 +32,7 @@
   [{:db/id        (d/id-literal :test -1)
     :db/ident     :x
     :db/unique    :db.unique/identity
-    :db/valueType :db.type/integer
+    :db/valueType :db.type/long
     :db.install/_attribute :db.part/db}
    ])