From 60c7db430113e9190935385073d39f582d41d20d Mon Sep 17 00:00:00 2001
From: Nick Alexander <nalexander@mozilla.com>
Date: Wed, 5 Oct 2016 15:29:40 -0700
Subject: [PATCH] Pre: Make testing consistent by sorting fulltext values
 before inserting.

This sorts fulltext values inserted in a single transaction, not across
transactions.  This makes the rowids assigned in the fulltext_values
table internally consistent, even as the order of entities and datoms
changes (as the transaction applying algorithm evolves over time).  The
test changes simply make the fulltext values sort easily.

In theory, these fulltext values could be very large, and sorting might
be very expensive.  In practice, we expect values to differ in their
first few characters, so that this is efficient (i.e., proportional to
the number of fulltext values inserted and not their size).
---
 src/common/datomish/db.cljc |  4 ++--
 test/datomish/db_test.cljc  | 42 ++++++++++++++++++-------------------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/common/datomish/db.cljc b/src/common/datomish/db.cljc
index f0d9c0d5..a76611f5 100644
--- a/src/common/datomish/db.cljc
+++ b/src/common/datomish/db.cljc
@@ -383,7 +383,7 @@
             (ref? a)           ; index_vaet
             (unique? a)        ; unique_value
             tag]]))
-      ops
+      (sort-by (fn [[_ _ _ v]] v) ops) ;; Make testing easier by sorting by string values.  TODO: discuss expense.
       (range initial-many-searchid 999999999))
     ["UPDATE fulltext_values SET searchid = NULL WHERE searchid IS NOT NULL"]))
 
@@ -412,7 +412,7 @@
               "INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0) VALUES "
               "(?, ?, (SELECT rowid FROM fulltext_values WHERE searchid = ?), ?, 0, ?)")
             e a searchid tx tag]]))
-      ops
+      (sort-by (fn [[_ _ _ v]] v) ops) ;; Make testing easier by sorting by string values.  TODO: discuss expense.
       (range initial-one-searchid 999999999))
     ["UPDATE fulltext_values SET searchid = NULL WHERE searchid IS NOT NULL"]))
 
diff --git a/test/datomish/db_test.cljc b/test/datomish/db_test.cljc
index dd8531bd..3cb47020 100644
--- a/test/datomish/db_test.cljc
+++ b/test/datomish/db_test.cljc
@@ -644,11 +644,11 @@
   (let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
 
     (let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/long [12345 123456]}
-                                         {:db/id (d/id-literal :db.part/user -1) :test/fulltext ["first fulltext value" "second fulltext value"]}]))
+                                         {:db/id (d/id-literal :db.part/user -1) :test/fulltext ["1 fulltext value" "2 fulltext value"]}]))
           eid1   (get-in report [:tempids (d/id-literal :db.part/user -1)])]
       (is (= (<? (<fulltext-values (d/db conn)))
-             [[1 "first fulltext value"]
-              [2 "second fulltext value"]]))
+             [[1 "1 fulltext value"]
+              [2 "2 fulltext value"]]))
       (is (= (<? (<datoms-after (d/db conn) tx0))
              #{[eid1 :test/fulltext 1]
                [eid1 :test/fulltext 2]
@@ -665,13 +665,13 @@
   (let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
 
     (testing "retractAttribute, fulltext"
-      (let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["first fulltext value" "second fulltext value"]}
+      (let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["1 fulltext value" "2 fulltext value"]}
                                            {:db/id (d/id-literal :db.part/user -2) :test/ref (d/id-literal :db.part/user -1)}]))
             eid1    (get-in report [:tempids (d/id-literal :db.part/user -1)])
             eid2    (get-in report [:tempids (d/id-literal :db.part/user -2)])]
         (is (= (<? (<fulltext-values (d/db conn)))
-               [[1 "first fulltext value"]
-                [2 "second fulltext value"]]))
+               [[1 "1 fulltext value"]
+                [2 "2 fulltext value"]]))
         (is (= (<? (<datoms-after (d/db conn) tx0))
                #{[eid1 :test/fulltext 1]
                  [eid1 :test/fulltext 2]
@@ -681,8 +681,8 @@
           (let [{tx1 :tx} (<? (d/<transact! conn [[:db.fn/retractAttribute eid2 :test/ref]]))]
             ;; fulltext values are not purged.
             (is (= (<? (<fulltext-values (d/db conn)))
-                   [[1 "first fulltext value"]
-                    [2 "second fulltext value"]]))
+                   [[1 "1 fulltext value"]
+                    [2 "2 fulltext value"]]))
             (is (= (<? (<datoms-after (d/db conn) tx0))
                    #{[eid1 :test/fulltext 1]
                      [eid1 :test/fulltext 2]})))
@@ -690,8 +690,8 @@
           (let [{tx2 :tx} (<? (d/<transact! conn [[:db.fn/retractAttribute eid1 :test/fulltext]]))]
             ;; fulltext values are not purged.
             (is (= (<? (<fulltext-values (d/db conn)))
-                   [[1 "first fulltext value"]
-                    [2 "second fulltext value"]]))
+                   [[1 "1 fulltext value"]
+                    [2 "2 fulltext value"]]))
             (is (= (<? (<datoms-after (d/db conn) tx0))
                    #{}))))))))
 
@@ -736,12 +736,12 @@
   (let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
 
     (let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/long [12345 123456]}
-                                         {:db/id (d/id-literal :db.part/user -2) :test/fulltext ["first fulltext value" "second fulltext value"]}]))
+                                         {:db/id (d/id-literal :db.part/user -2) :test/fulltext ["1 fulltext value" "2 fulltext value"]}]))
           eid1   (get-in report [:tempids (d/id-literal :db.part/user -1)])
           eid2   (get-in report [:tempids (d/id-literal :db.part/user -2)])]
       (is (= (<? (<fulltext-values (d/db conn)))
-             [[1 "first fulltext value"]
-              [2 "second fulltext value"]]))
+             [[1 "1 fulltext value"]
+              [2 "2 fulltext value"]]))
       (is (= (<? (<datoms-after (d/db conn) tx0))
              #{[eid2 :test/fulltext 1]
                [eid2 :test/fulltext 2]
@@ -758,16 +758,16 @@
   (let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
 
     (testing "retractEntity, fulltext"
-      (let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["first fulltext value" "second fulltext value"]}
+      (let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["1 fulltext value" "2 fulltext value"]}
                                            {:db/id (d/id-literal :db.part/user -2) :test/ref (d/id-literal :db.part/user -1)}
-                                           {:db/id (d/id-literal :db.part/user -3) :test/fulltext "other fulltext value"}]))
+                                           {:db/id (d/id-literal :db.part/user -3) :test/fulltext "3 fulltext value"}]))
             eid1    (get-in report [:tempids (d/id-literal :db.part/user -1)])
             eid2    (get-in report [:tempids (d/id-literal :db.part/user -2)])
             eid3    (get-in report [:tempids (d/id-literal :db.part/user -3)])]
         (is (= (<? (<fulltext-values (d/db conn)))
-               [[1 "first fulltext value"]
-                [2 "second fulltext value"]
-                [3 "other fulltext value"]]))
+               [[1 "1 fulltext value"]
+                [2 "2 fulltext value"]
+                [3 "3 fulltext value"]]))
         (is (= (<? (<datoms-after (d/db conn) tx0))
                #{[eid1 :test/fulltext 1]
                  [eid1 :test/fulltext 2]
@@ -786,9 +786,9 @@
           (let [{tx1 :tx} (<? (d/<transact! conn [[:db.fn/retractEntity eid3]]))]
             ;; fulltext values are not purged.
             (is (= (<? (<fulltext-values (d/db conn)))
-                   [[1 "first fulltext value"]
-                    [2 "second fulltext value"]
-                    [3 "other fulltext value"]]))
+                   [[1 "1 fulltext value"]
+                    [2 "2 fulltext value"]
+                    [3 "3 fulltext value"]]))
             (is (= (<? (<datoms-after (d/db conn) tx0))
                    #{[eid1 :test/fulltext 1]
                      [eid1 :test/fulltext 2]