greg/mentat
1
0
Fork 0
Code Issues 294 Pull requests Packages Projects Releases 9 Wiki Activity

Pre: Make testing consistent by sorting fulltext values before inserting.

Browse source 
This sorts fulltext values inserted in a single transaction, not across
transactions.  This makes the rowids assigned in the fulltext_values
table internally consistent, even as the order of entities and datoms
changes (as the transaction applying algorithm evolves over time).  The
test changes simply make the fulltext values sort easily.

In theory, these fulltext values could be very large, and sorting might
be very expensive.  In practice, we expect values to differ in their
first few characters, so that this is efficient (i.e., proportional to
the number of fulltext values inserted and not their size).
This commit is contained in:
Nick Alexander 2016-10-05 15:29:40 -07:00
parent d94dfae01b
commit 60c7db4301
2 changed files with 23 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/common/datomish/db.cljc
View file

@ -383,7 +383,7 @@
(ref? a) ; index_vaet
(unique? a) ; unique_value
tag]]))
ops
(sort-by (fn [[_ _ _ v]] v) ops) ;; Make testing easier by sorting by string values. TODO: discuss expense.
(range initial-many-searchid 999999999))
["UPDATE fulltext_values SET searchid = NULL WHERE searchid IS NOT NULL"]))
@ -412,7 +412,7 @@
"INSERT INTO temp.tx_lookup_before (e0, a0, v0, tx0, added0, value_type_tag0) VALUES "
"(?, ?, (SELECT rowid FROM fulltext_values WHERE searchid = ?), ?, 0, ?)")
e a searchid tx tag]]))
ops
(sort-by (fn [[_ _ _ v]] v) ops) ;; Make testing easier by sorting by string values. TODO: discuss expense.
(range initial-one-searchid 999999999))
["UPDATE fulltext_values SET searchid = NULL WHERE searchid IS NOT NULL"]))

42
test/datomish/db_test.cljc
View file

@ -644,11 +644,11 @@
(let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
(let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/long [12345 123456]}
{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["first fulltext value" "second fulltext value"]}]))
{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["1 fulltext value" "2 fulltext value"]}]))
eid1 (get-in report [:tempids (d/id-literal :db.part/user -1)])]
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{[eid1 :test/fulltext 1]
[eid1 :test/fulltext 2]
@ -665,13 +665,13 @@
(let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
(testing "retractAttribute, fulltext"
(let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["first fulltext value" "second fulltext value"]}
(let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["1 fulltext value" "2 fulltext value"]}
{:db/id (d/id-literal :db.part/user -2) :test/ref (d/id-literal :db.part/user -1)}]))
eid1 (get-in report [:tempids (d/id-literal :db.part/user -1)])
eid2 (get-in report [:tempids (d/id-literal :db.part/user -2)])]
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{[eid1 :test/fulltext 1]
[eid1 :test/fulltext 2]
@ -681,8 +681,8 @@
(let [{tx1 :tx} (<? (d/<transact! conn [[:db.fn/retractAttribute eid2 :test/ref]]))]
;; fulltext values are not purged.
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{[eid1 :test/fulltext 1]
[eid1 :test/fulltext 2]})))
@ -690,8 +690,8 @@
(let [{tx2 :tx} (<? (d/<transact! conn [[:db.fn/retractAttribute eid1 :test/fulltext]]))]
;; fulltext values are not purged.
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{}))))))))
@ -736,12 +736,12 @@
(let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
(let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/long [12345 123456]}
{:db/id (d/id-literal :db.part/user -2) :test/fulltext ["first fulltext value" "second fulltext value"]}]))
{:db/id (d/id-literal :db.part/user -2) :test/fulltext ["1 fulltext value" "2 fulltext value"]}]))
eid1 (get-in report [:tempids (d/id-literal :db.part/user -1)])
eid2 (get-in report [:tempids (d/id-literal :db.part/user -2)])]
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{[eid2 :test/fulltext 1]
[eid2 :test/fulltext 2]
@ -758,16 +758,16 @@
(let [tx0 (:tx (<? (d/<transact! conn retract-schema)))]
(testing "retractEntity, fulltext"
(let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["first fulltext value" "second fulltext value"]}
(let [report (<? (d/<transact! conn [{:db/id (d/id-literal :db.part/user -1) :test/fulltext ["1 fulltext value" "2 fulltext value"]}
{:db/id (d/id-literal :db.part/user -2) :test/ref (d/id-literal :db.part/user -1)}
{:db/id (d/id-literal :db.part/user -3) :test/fulltext "other fulltext value"}]))
{:db/id (d/id-literal :db.part/user -3) :test/fulltext "3 fulltext value"}]))
eid1 (get-in report [:tempids (d/id-literal :db.part/user -1)])
eid2 (get-in report [:tempids (d/id-literal :db.part/user -2)])
eid3 (get-in report [:tempids (d/id-literal :db.part/user -3)])]
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]
[3 "other fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]
[3 "3 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{[eid1 :test/fulltext 1]
[eid1 :test/fulltext 2]
@ -786,9 +786,9 @@
(let [{tx1 :tx} (<? (d/<transact! conn [[:db.fn/retractEntity eid3]]))]
;; fulltext values are not purged.
(is (= (<? (<fulltext-values (d/db conn)))
[[1 "first fulltext value"]
[2 "second fulltext value"]
[3 "other fulltext value"]]))
[[1 "1 fulltext value"]
[2 "2 fulltext value"]
[3 "3 fulltext value"]]))
(is (= (<? (<datoms-after (d/db conn) tx0))
#{[eid1 :test/fulltext 1]
[eid1 :test/fulltext 2]