From fc845a995066a0a773519908f891282e4e90183c Mon Sep 17 00:00:00 2001 From: Richard Newman Date: Thu, 4 Aug 2016 18:50:34 -0700 Subject: [PATCH] Implement basic fulltext binding. r=nalexander --- src/datomish/query/clauses.cljc | 6 +- src/datomish/query/functions.cljc | 175 ++++++++++++++++++++++++++++++ src/datomish/query/source.cljc | 21 +++- test/datomish/test/query.cljc | 3 +- 4 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 src/datomish/query/functions.cljc diff --git a/src/datomish/query/clauses.cljc b/src/datomish/query/clauses.cljc index 54e2177e..69920f41 100644 --- a/src/datomish/query/clauses.cljc +++ b/src/datomish/query/clauses.cljc @@ -5,6 +5,7 @@ (ns datomish.query.clauses (:require [datomish.query.cc :as cc] + [datomish.query.functions :as functions] [datomish.query.source :refer [attribute-in-source constant-in-source @@ -170,8 +171,9 @@ ;; TODO: handle And within the Or patterns. (raise "Non-simple `or` clauses not yet supported." {:clause orc}))) -(apply-function-clause [cc function] - cc) +(defn apply-function-clause [cc function] + (or (functions/apply-sql-function cc function) + (raise "Unknown function expression." {:clause function}))) ;; We're keeping this simple for now: a straightforward type switch. (defn apply-clause [cc it] diff --git a/src/datomish/query/functions.cljc b/src/datomish/query/functions.cljc new file mode 100644 index 00000000..141b0f3e --- /dev/null +++ b/src/datomish/query/functions.cljc @@ -0,0 +1,175 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. + +(ns datomish.query.functions + (:require + [honeysql.format :as fmt] + [datomish.query.cc :as cc] + [datomish.query.source :as source] + [datomish.util :as util #?(:cljs :refer-macros :clj :refer) [raise raise-str cond-let]] + [datascript.parser :as dp + #?@(:cljs + [:refer + [ + BindColl + BindScalar + BindTuple + BindIgnore + Constant + Function + PlainSymbol + SrcVar + Variable + ]])] + [honeysql.core :as sql] + [clojure.string :as str] + ) + #?(:clj + (:import + [datascript.parser + BindColl + BindScalar + BindTuple + BindIgnore + Constant + Function + PlainSymbol + SrcVar + Variable + ]))) + +;; honeysql's MATCH handler doesn't work for sqlite. This does. +(defmethod fmt/fn-handler "match" [_ col val] + (str (fmt/to-sql col) " MATCH " (fmt/to-sql val))) + +(defn fulltext-attribute? [source attribute] + ;; TODO: schema lookup. + true) + +(defn bind-coll->binding-vars [bind-coll] + (:bindings (:binding bind-coll))) + +(defn binding-placeholder-or-variable? [binding] + (or + ;; It's a placeholder... + (instance? BindIgnore binding) + + ;; ... or it's a scalar binding to a variable. + (and + (instance? BindScalar binding) + (instance? Variable (:variable binding))))) + +(defn- validate-fulltext-clause [cc function] + (let [bind-coll (:binding function) + [src attr search] (:args function)] + (when-not (and (instance? SrcVar src) + (= "$" (name (:symbol src)))) + (raise "Non-default sources not supported." {:arg src})) + (when-not (instance? Constant attr) + (raise "Non-constant fulltext attributes not supported." {:arg attr})) + + (when-not (fulltext-attribute? (:source cc) (:value attr)) + (raise-str "Attribute " (:value attr) " is not a fulltext-indexed attribute.")) + + (when-not (and (instance? BindColl bind-coll) + (instance? BindTuple (:binding bind-coll)) + (every? binding-placeholder-or-variable? + (bind-coll->binding-vars bind-coll))) + + (raise "Unexpected binding value." {:binding bind-coll})))) + +(defn apply-fulltext-clause [cc function] + (validate-fulltext-clause cc function) + + ;; A fulltext search string is either a constant string or a variable binding. + ;; The search string and the attribute are used to generate a SQL MATCH expression: + ;; table MATCH 'search string' + ;; This is then joined against an ordinary pattern to yield entity, value, and tx. + ;; We do not currently support scoring; the score value will always be 0. + (let [[src attr search] (:args function) + + ;; Pull out the symbols for the binding array. + [entity value tx score] + (map (comp :symbol :variable) ; This will nil-out placeholders. + (get-in function [:binding :binding :bindings])) + + ;; Find the FTS table name and alias. We might have multiple fulltext + ;; expressions so we will generate a query like + ;; SELECT ttt.a FROM t1 AS ttt WHERE ttt.t1 MATCH 'string' + [fulltext-table fulltext-alias] (source/source->fulltext-from (:source cc)) ; [:t1 :ttt] + match-column (sql/qualify fulltext-alias fulltext-table) ; :ttt.t1 + match-value (cc/argument->value cc search) + + [datom-table datom-alias] (source/source->non-fulltext-from (:source cc)) + + ;; The following will end up being added to the CC. + from [[fulltext-table fulltext-alias] + [datom-table datom-alias]] + + wheres [[:match match-column match-value] ; The FTS match. + + ;; The fulltext rowid-to-datom correspondence. + [:= + (sql/qualify datom-alias :v) + (sql/qualify fulltext-alias :rowid)] + + ;; The attribute itself must match. + [:= + (sql/qualify datom-alias :a) + (source/attribute-in-source (:source cc) (:value attr))]] + + ;; Now compose any bindings for entity, value, tx, and score. + ;; TODO: do we need to examine existing bindings to capture + ;; wheres for any of these? We shouldn't, because the CC will + ;; be internally cross-where'd when everything is done... + bindings (into {} + (filter + (comp not nil? first) + [[entity [(sql/qualify datom-alias :e)]] + [value [match-column]] + [tx [(sql/qualify datom-alias :tx)]] + + ;; Future: use matchinfo to compute a score + ;; if this is a variable rather than a placeholder. + [score [0]]]))] + + (cc/augment-cc cc from bindings wheres))) + +(def sql-functions + ;; Future: versions of this that uses snippet() or matchinfo(). + {"fulltext" apply-fulltext-clause}) + +(defn apply-sql-function + "Either returns an application of `function` to `cc`, or nil to + encourage you to try a different application." + [cc function] + (when (and (instance? Function function) + (instance? PlainSymbol (:fn function))) + (when-let [apply-f (get sql-functions (name (:symbol (:fn function))))] + (apply-f cc function)))) + +;; A fulltext expression parses to: +;; +;; Function ( :fn, :args ) +;; +;; The args begin with a SrcVar, and then are attr and search. +;; +;; This binds a relation of [?entity ?value ?tx ?score]: +;; +;; BindColl +;; :binding BindTuple +;; :bindings [BindScalar...] +;; +;; #datascript.parser.Function +;; {:fn #datascript.parser.PlainSymbol{:symbol fulltext}, +;; :args [#datascript.parser.SrcVar{:symbol $} +;; #datascript.parser.Constant{:value :artist/name} +;; #datascript.parser.Variable{:symbol ?search}], +;; :binding #datascript.parser.BindColl +;; {:binding #datascript.parser.BindTuple +;; {:bindings [ +;; #datascript.parser.BindScalar{:variable #datascript.parser.Variable{:symbol ?entity}} +;; #datascript.parser.BindScalar{:variable #datascript.parser.Variable{:symbol ?name}} +;; #datascript.parser.BindScalar{:variable #datascript.parser.Variable{:symbol ?tx}} +;; #datascript.parser.BindScalar{:variable #datascript.parser.Variable{:symbol ?score}}]}}} diff --git a/src/datomish/query/source.cljc b/src/datomish/query/source.cljc index f69d3e59..161f760d 100644 --- a/src/datomish/query/source.cljc +++ b/src/datomish/query/source.cljc @@ -35,6 +35,9 @@ (defprotocol Source (source->from [source attribute] "Returns a pair, `[table alias]` for a pattern with the provided attribute.") + (source->non-fulltext-from [source]) + (source->fulltext-from [source] + "Returns a pair, `[table alias]` for querying the source's fulltext index.") (source->constraints [source alias]) (attribute-in-source [source attribute]) (constant-in-source [source constant])) @@ -42,7 +45,8 @@ (defrecord DatomsSource [table ; Typically :datoms. - fts-view ; Typically :fulltext_datoms. + fulltext-table ; Typically :fulltext_values + fulltext-view ; Typically :fulltext_datoms. columns ; e.g., [:e :a :v :tx] ;; `attribute-transform` is a function from attribute to constant value. Used to @@ -51,7 +55,7 @@ ;; turn, e.g., the literal 'true' into 1. attribute-transform constant-transform - + ;; `table-alias` is a function from table to alias, e.g., :datoms => :datoms1234. table-alias @@ -69,7 +73,15 @@ (:table source) ;; It's variable. We must act as if it could be a fulltext datom. - (:fts-view source))] + (:fulltext-view source))] + [table ((:table-alias source) table)])) + + (source->non-fulltext-from [source] + (let [table (:table source)] + [table ((:table-alias source) table)])) + + (source->fulltext-from [source] + (let [table (:fulltext-table source)] [table ((:table-alias source) table)])) (source->constraints [source alias] @@ -85,7 +97,8 @@ (defn datoms-source [db] (map->DatomsSource {:table :datoms - :fts-view :fulltext_datoms + :fulltext-table :fulltext_values + :fulltext-view :fulltext_datoms :columns [:e :a :v :tx :added] :attribute-transform transforms/attribute-transform-string :constant-transform transforms/constant-transform-default diff --git a/test/datomish/test/query.cljc b/test/datomish/test/query.cljc index 2518a5a6..0abd6df0 100644 --- a/test/datomish/test/query.cljc +++ b/test/datomish/test/query.cljc @@ -28,7 +28,8 @@ (defn mock-source [db] (source/map->DatomsSource {:table :datoms - :fts-view :fulltext_datoms + :fulltext-table :fulltext_values + :fulltext-view :fulltext_datoms :columns [:e :a :v :tx :added] :attribute-transform transforms/attribute-transform-string :constant-transform transforms/constant-transform-default