greg/mentat
1
0
Fork 0
Code Issues 294 Pull requests Projects Releases 9 Packages Wiki Activity

Make Datomish work in a Firefox add-on on top of Sqlite.jsm. Fixes #48. r=nalexander

Browse source
This commit is contained in:
Richard Newman 2016-09-09 09:26:19 -07:00
commit ac253bfea0
37 changed files with 421 additions and 175 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.gitignore vendored
View file

@ -22,5 +22,11 @@ pom.xml
pom.xml.asc
/.cljs_node_repl/
/.cljs_rhino_repl/
/release-js/datomish.js
/release-js/datomish.bare.js
/release-browser
/release-browser/datomish.js
/release-browser/datomish.bare.js
/release-node
/release-node/datomish.js
/release-node/datomish.bare.js
/addon/datomish-test.xpi
/addon/datomish.js

2
addon/README.md Normal file
View file

@ -0,0 +1,2 @@
#Datomish Test
An example add-on that loads Datomish on top of Sqlite.jsm.

15
addon/index.js Normal file
View file

@ -0,0 +1,15 @@
var self = require("sdk/self");
console.log("Datomish Test");
console.log("This: " + this);
var datomish = require("datomish.js");
datomish.open("/tmp/foobar.db").then(function (db) {
console.log("Got " + db);
try {
db.close();
console.log("Closed.");
} catch (e) {
console.log("Couldn't close: " + e);
}
});

15
addon/package.json Normal file
View file

@ -0,0 +1,15 @@
{
"title": "Datomish Test",
"name": "datomish-test",
"version": "0.0.1",
"description": "An example add-on that loads Datomish on top of Sqlite.jsm.",
"main": "index.js",
"author": "Richard Newman <rnewman@mozilla.com>",
"engines": {
"firefox": ">=48.0a1"
},
"license": "MPL-2.0",
"keywords": [
"jetpack"
]
}

141
project.clj
View file

@ -3,7 +3,7 @@
:url "https://github.com/mozilla/datomish"
:license {:name "Mozilla Public License Version 2.0"
:url "https://github.com/mozilla/datomish/blob/master/LICENSE"}
:dependencies [[org.clojure/clojurescript "1.9.89"]
:dependencies [[org.clojure/clojurescript "1.9.229"]
[org.clojure/clojure "1.8.0"]
[org.clojure/core.async "0.2.385"]
[datascript "0.15.1"]
@ -11,37 +11,89 @@
[com.taoensso/tufte "1.0.2"]
[jamesmacaulay/cljs-promises "0.1.0"]]
:cljsbuild {:builds {:release {
:source-paths ["src"]
:assert false
:compiler {:output-to "release-js/datomish.bare.js"
:optimizations :advanced
:pretty-print false
:elide-asserts true
:output-wrapper false
:parallel-build true}
:notify-command ["release-js/wrap_bare.sh"]}
:advanced {:source-paths ["src"]
:compiler {:output-to "target/advanced/datomish.js"
:optimizations :advanced
:source-map "target/advanced/datomish.js.map"
:pretty-print true
:recompile-dependents true
:parallel-build true
}}
:test {
:source-paths ["src" "test"]
:compiler {:output-to "target/test/datomish.js"
:output-dir "target/test"
:main datomish.test
:optimizations :none
:source-map true
:recompile-dependents true
:parallel-build true
:target :nodejs
}}
}
}
:cljsbuild {:builds
{
:release-node
{
:source-paths ["src-node" "src"]
:assert false
:compiler
{
:elide-asserts true
:hashbang false
:language-in :ecmascript5
:language-out :ecmascript5
:optimizations :advanced
:output-dir "release-node"
:output-to "release-node/datomish.bare.js"
:output-wrapper false
:parallel-build true
:pretty-print false
:target :nodejs
}
:notify-command ["release-node/wrap_bare.sh"]}
:release-browser
;; Release builds for use in Firefox must:
;; * Use :optimizations > :none, so that a single file is generated
;; without a need to import Closure's own libs.
;; * Be wrapped, so that a CommonJS module is produced.
;; * Have a preload script that defines what `println` does.
;;
;; There's no point in generating a source map -- it'll be wrong
;; due to wrapping.
{
:source-paths ["src-browser" "src"]
:assert false
:compiler
{
:elide-asserts true
:externs ["src-browser/externs.js"]
:language-in :ecmascript5
:language-out :ecmascript5
:optimizations :advanced
:output-dir "release-browser"
:output-to "release-browser/datomish.bare.js"
:output-wrapper false
:parallel-build true
:preloads [datomish.preload]
:pretty-print true
:pseudo-names true
:static-fns true
}
:notify-command ["release-browser/wrap_bare.sh"]}
:advanced
{:source-paths ["src-node" "src"]
:compiler
{
:language-in :ecmascript5
:language-out :ecmascript5
:output-dir "target/advanced"
:output-to "target/advanced/datomish.js"
:optimizations :advanced
:parallel-build true
:pretty-print true
:source-map "target/advanced/datomish.js.map"
:target :nodejs
}}
:test
{
:source-paths ["src-node" "src" "test"]
:compiler
{
:language-in :ecmascript5
:language-out :ecmascript5
:main datomish.test
:optimizations :none
:output-dir "target/test"
:output-to "target/test/datomish.js"
:parallel-build true
:source-map true
:target :nodejs
}}
}}
:profiles {:dev {:dependencies [[cljsbuild "1.1.3"]
[tempfile "0.2.0"]
@ -59,7 +111,26 @@
:doo {:build "test"}
:clean-targets ^{:protect false} ["target"
"release-js/datomish.bare.js"
"release-js/datomish.js"]
:clean-targets ^{:protect false}
[
"target"
"release-node/cljs/"
"release-node/cljs_promises/"
"release-node/clojure/"
"release-node/datascript/"
"release-node/datomish/"
"release-node/honeysql/"
"release-node/taoensso/"
"release-node/datomish.bare.js"
"release-node/datomish.js"
"release-browser/cljs/"
"release-browser/cljs_promises/"
"release-browser/clojure/"
"release-browser/datascript/"
"release-browser/datomish/"
"release-browser/honeysql/"
"release-browser/taoensso/"
"release-browser/datomish.bare.js"
"release-browser/datomish.js"
]
)

0
release-js/README.md → release-browser/README.md
View file

0
release-js/test_include_node.js → release-browser/test_include_node.js
View file

7
release-browser/wrap_bare.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/sh
set -e
(cat release-browser/wrapper.prefix; cat release-browser/datomish.bare.js; cat release-browser/wrapper.suffix) > release-browser/datomish.js
echo "Packed release-browser/datomish.js"

34
release-browser/wrapper.prefix Normal file
View file

@ -0,0 +1,34 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
// Datomish 0.1.0-SNAPSHOT
(function (definition) {
// This file will function properly as a <script> tag, or a module
// using CommonJS and NodeJS or RequireJS module formats.
// Wrapper gratefully adapted from:
// https://github.com/kriskowal/q/blob/v1/q.js
// https://github.com/swannodette/mori/blob/master/support/wrapper.js
// https://github.com/tonsky/datascript/blob/master/release-js/wrapper.js
// CommonJS
if (typeof exports === "object") {
module.exports = definition();
// RequireJS
} else if (typeof define === "function" && define.amd) {
define(definition);
// <script>
} else {
datomish = definition();
}
})(function () {
return function () {
// Monkeypatch setTimeout so that the Closure Compiler
// output can use it in a Sandbox context.
var { setTimeout } = require("sdk/timers");
this.setTimeout = setTimeout;

0
release-js/wrapper.suffix → release-browser/wrapper.suffix
View file

7
release-js/wrap_bare.sh
View file

@ -1,7 +0,0 @@
#!/bin/sh
set -e
(cat release-js/wrapper.prefix; cat release-js/datomish.bare.js; cat release-js/wrapper.suffix) > release-js/datomish.js
echo "Packed release-js/datomish.js"

5
release-node/README.md Normal file
View file

@ -0,0 +1,5 @@
# Datomish
Datomish is a persistent, embedded knowledge base. It's written in ClojureScript, and draws heavily on [DataScript](https://github.com/tonsky/datascript) and [Datomic](http://datomic.com).
For more info, check out the [project page](https://github.com/mozila/datomish).

2
release-node/test_include_node.js Normal file
View file

@ -0,0 +1,2 @@
var d = require('./datomish');
console.log(d.q("[:find ?e ?v :where [?e \"name\" ?v] {:x :y}]"));

7
release-node/wrap_bare.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/sh
set -e
(cat release-node/wrapper.prefix && cat release-node/datomish.bare.js && cat release-node/wrapper.suffix) > release-node/datomish.js
echo "Packed release-node/datomish.js"

0
release-js/wrapper.prefix → release-node/wrapper.prefix
View file

6
release-node/wrapper.suffix Normal file
View file

@ -0,0 +1,6 @@
;return this.datomish.js;
}.call({});
});

13
src-browser/datomish/core.cljs Normal file
View file

@ -0,0 +1,13 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.core
(:require
[honeysql.format :as sql]
[datomish.db :as db]
[datomish.db-factory :as db-factory]
[datomish.js-sqlite :as js-sqlite]
[datomish.sqlite :as sqlite]
[datomish.transact :as transact]))

20
src-browser/datomish/js_sqlite.cljs Normal file
View file

@ -0,0 +1,20 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js-sqlite
(:require
[datomish.sqlite :as s]
[datomish.js-util :refer [is-node?]]
[datomish.sqlitejsm-sqlite :as sqlitejsm-sqlite]))
(def open sqlitejsm-sqlite/open)
(extend-protocol s/ISQLiteConnectionFactory
string
(<sqlite-connection [path]
(open path))
object
(<sqlite-connection [tempfile]
(open (.-name tempfile))))

8
src-browser/datomish/preload.cljs Normal file
View file

@ -0,0 +1,8 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.preload)
(enable-console-print!)
(println "Console printing enabled.")

57
src-browser/datomish/sqlitejsm_sqlite.cljs Normal file
View file

@ -0,0 +1,57 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.sqlitejsm-sqlite
(:require
[cljs-promises.async]
[datomish.sqlite :as s]))
(def sqlite (.import (aget (js/require "chrome") "Cu") "resource://gre/modules/Sqlite.jsm"))
(println "sqlite is" (pr-str sqlite))
;; mozIStorageRow instances expose two methods: getResultByIndex and getResultByName.
;; Our code expects to treat rows as associative containers, from keyword to value.
;; So we implement ILookup (which has a different signature for ClojureScript than
;; Clojure!), hope that we handle nil/NULL correctly, and switch between integers
;; and keywords.
(deftype
StorageRow
[row]
ILookup
(-lookup [o k]
(-lookup o k nil))
(-lookup [o k not-found]
(or (if (integer? k)
(.getResultByIndex row k)
(.getResultByName row (clj->js (name k))))
not-found)))
(defrecord SQLite3Connection [db]
s/ISQLiteConnection
(-execute!
[db sql bindings]
(cljs-promises.async/pair-port
(.execute (.-db db) sql (or (clj->js bindings) #js []))))
(-each
[db sql bindings row-cb]
(let [cb (fn [row]
(row-cb (StorageRow. row)))]
(cljs-promises.async/pair-port
(.execute (.-db db) sql (or (clj->js bindings) #js []) (when row-cb cb)))))
(close
[db]
(cljs-promises.async/pair-port
(.close (.-db db)))))
(defn open
[path & {:keys [mode] :or {mode 6}}]
(cljs-promises.async/pair-port
(->
(.openConnection (aget sqlite "Sqlite") (clj->js {:path path :sharedMemoryCache false}))
(.then ->SQLite3Connection))))

33
src-browser/externs.js Normal file
View file

@ -0,0 +1,33 @@
var SqliteStatic = {};
/**
* @param {Object} options
* @return {Promise.<Sqlite>}
*/
SqliteStatic.openConnection = function (options) {}
var Sqlite = {}
/**
* @param {string} sql
* @param {Array} bindings
* @return {Promise}
*/
Sqlite.execute = function (sql, bindings) {}
/**
* @return {Promise}
*/
Sqlite.close = function() {}
var StorageRow = {};
/**
* @param {string} index
*/
StorageRow.getResultByIndex = function (index) {}
/**
* @param {string} name
*/
StorageRow.getResultByName = function (name) {}

0
src/datomish/core.cljs → src-node/datomish/core.cljs
View file

20
src-node/datomish/js_sqlite.cljs Normal file
View file

@ -0,0 +1,20 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js-sqlite
(:require
[datomish.sqlite :as s]
[datomish.js-util :refer [is-node?]]
[datomish.promise-sqlite :as promise-sqlite]))
(def open promise-sqlite/open)
(extend-protocol s/ISQLiteConnectionFactory
string
(<sqlite-connection [path]
(open path))
object ;; TODO: narrow this to the result of node-tempfile/tempfile.
(<sqlite-connection [tempfile]
(open (.-name tempfile))))

0
src/datomish/node_tempfile.cljs → src-node/datomish/node_tempfile.cljs
View file

0
src/datomish/node_tempfile_macros.cljc → src-node/datomish/node_tempfile_macros.cljc
View file

9
src/datomish/promise_sqlite.cljs → src-node/datomish/promise_sqlite.cljs
View file

@ -35,12 +35,3 @@
(->
(.open sqlite.DB path (clj->js {:mode mode}))
(.then ->SQLite3Connection))))
(extend-protocol s/ISQLiteConnectionFactory
string
(<sqlite-connection [path]
(open path))
object ;; TODO: narrow this to the result of node-tempfile/tempfile.
(<sqlite-connection [tempfile]
(open (.-name tempfile))))

111
src/datomish/exec_repl.cljc
View file

@ -1,111 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.exec-repl
#?(:cljs
(:require-macros
[datomish.util :refer [while-let]]
[datomish.pair-chan :refer [go-pair <?]]
[cljs.core.async.macros :refer [go]]))
(:require
[datomish.db-factory]
[datomish.db :as db]
[datomish.sqlite :as s]
[datomish.sqlite-schema :as ss]
[datomish.query :as dq]
#?@(:clj
[[datomish.jdbc-sqlite]
[datomish.pair-chan :refer [go-pair <?]]
[datomish.util :refer [while-let]]
[clojure.core.async :refer [<!]]])
#?@(:cljs
[[datomish.promise-sqlite]
[datomish.pair-chan]
[datomish.util]
[cljs.core.async :refer [<!]]])))
#?(:clj
(defn pair-channel->lazy-seq
"Returns a blocking lazy sequence of items taken from the provided channel."
[channel]
(lazy-seq
(when-let [v (clojure.core.async/<!! channel)]
(if (second v)
(cons v nil)
(cons v (pair-channel->lazy-seq channel)))))))
#?(:clj
(defn run-to-pair-seq
"Given an open database, returns a lazy sequence of results.
When fully consumed, underlying resources will be released."
[db find]
(pair-channel->lazy-seq (db/<?run db find))))
#_
(defn xxopen []
(datomish.pair-chan/go-pair
(let [d (datomish.pair-chan/<? (s/<sqlite-connection "/tmp/import.sqlite"))]
(clojure.core.async/<!! (ss/<ensure-current-version d))
(def db d))))
;; With an open DB…
#_(run-to-pair-seq
db
'[:find ?page :in $ :where [?page :page/starred true ?t]])
;; In a Clojure REPL with no open DB…
#_(clojure.core.async/<!!
(datomish.exec-repl/<open-and-run-to-seq-promise
"/tmp/foo.sqlite"
'[:find ?page :in $ :where [?page :page/starred true ?t]]))
#_(require 'datomish.exec-repl)
#_(in-ns 'datomish.exec-repl)
#_
(go-pair
(let [connection (<? (s/<sqlite-connection "/tmp/bigport.db"))
d (<? (datomish.db-factory/<db-with-sqlite-connection connection))]
(def db d)))
#_
(go-pair
(println (count (first (time
(<! (db/<?q db
'[:find ?url ?title :in $ :where
[?page :page/visitAt ?v] [(> ?v 1438748166567751)] [?page :page/title ?title] [?page :page/url ?url] ] {})))))))
#_
(go-pair
(let [connection (<? (s/<sqlite-connection "/tmp/foo.sqlite"))
dd (<? (datomish.db-factory/<db-with-sqlite-connection connection))]
(def *db* dd)))
#_
(clojure.core.async/<!!
(go-pair
(let [now -1
forms (mapcat (fn [i]
(map (fn [j]
[:db/add i :x j true])
(range 1000 (* i 2000) i)))
(range 1 10))]
(println "Adding" (count forms) "forms")
(<? (transact/<transact! *db* forms nil now)))))
#_
(go-pair
(let [connection (<? (s/<sqlite-connection "/tmp/foo.sqlite"))
dd (<? (db/<with-sqlite-connection connection))]
(println
(count
(<? (db/<?q dd
'[:find ?e ?v :in $ :where
[?e :x ?v]
#_[(> ?v 1000)]] {}))))))
(dq/parse '[:find ?entity ?tx ?score
:in $ ?search
:where [(foobar $ :artist/name ?search) [[?entity _ ?tx ?score]]]])
(honeysql.core/format {:select [:?foo] :from [:foo] :where [:match :foo.x "Bar"]})

40
src/datomish/js.cljs
View file

@ -1,11 +1,47 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js
(:refer-clojure :exclude [])
(:require-macros
[datomish.pair-chan :refer [go-pair <?]])
(:require
[datomish.core :as d]
[cljs.reader]))
[cljs.core.async :as a :refer [take! <! >!]]
[cljs.reader]
[cljs-promises.core :refer [promise]]
[datomish.db :as db]
[datomish.db-factory :as db-factory]
[datomish.pair-chan]
[datomish.sqlite :as sqlite]
[datomish.js-sqlite :as js-sqlite]
[datomish.transact :as transact]))
(defn- take-pair-as-promise! [ch]
;; Just like take-as-promise!, but aware that it's handling a pair channel.
(promise
(fn [resolve reject]
(letfn [(split-pair [[v e]]
(if e
(reject e)
(resolve v)))]
(cljs.core.async/take! ch split-pair)))))
;; Public API.
(defn ^:export open [path]
;; Eventually, URI. For now, just a plain path (no file://).
(take-pair-as-promise!
(go-pair
(let [conn (<? (sqlite/<sqlite-connection path))
db (<? (db-factory/<db-with-sqlite-connection conn))]
(let [c (transact/connection-with-db db)]
(clj->js
{:conn c
:close (fn [] (db/close-db db))
:toString (fn [] (str "#<DB " path ">"))
:path path}))))))
(defn ^:export q [query & sources]
(let [query (cljs.reader/read-string query)]
(clj->js query)))

14
src/datomish/js_util.cljs Normal file
View file

@ -0,0 +1,14 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js-util)
(defn is-node? []
(try
(= "[object process]"
(.toString (aget js/global "process")))
(catch js/ReferenceError e
false)
(catch js/TypeError e
false)))

4
src/datomish/query/clauses.cljc
View file

@ -126,8 +126,8 @@
"Transform a DataScript Pattern instance into the parts needed
to build a SQL expression.
@arg cc A CC instance.
@arg pattern The pattern instance.
@param cc A CC instance.
@param pattern The pattern instance.
@return an augmented CC"
[cc pattern]
(when-not (instance? Pattern pattern)

4
src/datomish/sqlite.cljc
View file

@ -101,7 +101,9 @@
(defn get-user-version [db]
(go-pair
(let [row (first (<? (all-rows db ["PRAGMA user_version"])))]
(:user_version row))))
(or
(:user_version row)
0))))
(defn set-user-version [db version]
(execute! db [(str "PRAGMA user_version = " version)]))

2
test/datomish/db_test.cljc
View file

@ -22,7 +22,7 @@
[datomish.test-macros :refer [deftest-async deftest-db]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async deftest-db]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/places/import_test.cljc
View file

@ -21,7 +21,7 @@
[datomish.test-macros :refer [deftest-async]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/promise_sqlite_test.cljs
View file

@ -14,7 +14,7 @@
[cljs.test :refer-macros [is are deftest testing async]]
[datomish.pair-chan]
[datomish.sqlite :as s]
[datomish.promise-sqlite :as ps]))
[datomish.js-sqlite :as ps]))
(deftest-async test-all-rows
(with-tempfile [t (tempfile)]

2
test/datomish/query_test.cljc
View file

@ -16,7 +16,7 @@
[datomish.test-macros :refer [deftest-async]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/test/query.cljc
View file

@ -22,7 +22,7 @@
[tempfile.core :refer [tempfile with-tempfile]]
[clojure.test :as t :refer [is are deftest testing]]])
#?@(:cljs
[[datomish.promise-sqlite]
[[datomish.js-sqlite]
[datomish.test-macros :refer-macros [deftest-db]]
[honeysql.core :as sql :refer-macros [param]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/tofinoish_test.cljc
View file

@ -17,7 +17,7 @@
[datomish.test-macros :refer [deftest-async deftest-db]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async deftest-db]]
[datomish.node-tempfile :refer [tempfile]]