greg/mentat
1
0
Fork 0
Code Issues 294 Pull requests Packages Projects Releases 9 Wiki Activity

Make Datomish work in a Firefox add-on on top of Sqlite.jsm. Fixes #48. r=nalexander

Browse source
This commit is contained in:
Richard Newman 2016-09-09 09:26:19 -07:00
parent 21f672f921 ee2dca05d0
commit ac253bfea0
37 changed files with 421 additions and 175 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.gitignore vendored
View file

@ -22,5 +22,11 @@ pom.xml
pom.xml.asc
/.cljs_node_repl/
/.cljs_rhino_repl/
/release-js/datomish.js
/release-js/datomish.bare.js
/release-browser
/release-browser/datomish.js
/release-browser/datomish.bare.js
/release-node
/release-node/datomish.js
/release-node/datomish.bare.js
/addon/datomish-test.xpi
/addon/datomish.js

2
addon/README.md Normal file
View file

@ -0,0 +1,2 @@
#Datomish Test
An example add-on that loads Datomish on top of Sqlite.jsm.

15
addon/index.js Normal file
View file

@ -0,0 +1,15 @@
var self = require("sdk/self");
console.log("Datomish Test");
console.log("This: " + this);
var datomish = require("datomish.js");
datomish.open("/tmp/foobar.db").then(function (db) {
console.log("Got " + db);
try {
db.close();
console.log("Closed.");
} catch (e) {
console.log("Couldn't close: " + e);
}
});

15
addon/package.json Normal file
View file

@ -0,0 +1,15 @@
{
"title": "Datomish Test",
"name": "datomish-test",
"version": "0.0.1",
"description": "An example add-on that loads Datomish on top of Sqlite.jsm.",
"main": "index.js",
"author": "Richard Newman <rnewman@mozilla.com>",
"engines": {
"firefox": ">=48.0a1"
},
"license": "MPL-2.0",
"keywords": [
"jetpack"
]
}

141
project.clj
View file

@ -3,7 +3,7 @@
:url "https://github.com/mozilla/datomish"
:license {:name "Mozilla Public License Version 2.0"
:url "https://github.com/mozilla/datomish/blob/master/LICENSE"}
:dependencies [[org.clojure/clojurescript "1.9.89"]
:dependencies [[org.clojure/clojurescript "1.9.229"]
[org.clojure/clojure "1.8.0"]
[org.clojure/core.async "0.2.385"]
[datascript "0.15.1"]
@ -11,37 +11,89 @@
[com.taoensso/tufte "1.0.2"]
[jamesmacaulay/cljs-promises "0.1.0"]]
:cljsbuild {:builds {:release {
:source-paths ["src"]
:assert false
:compiler {:output-to "release-js/datomish.bare.js"
:optimizations :advanced
:pretty-print false
:elide-asserts true
:output-wrapper false
:parallel-build true}
:notify-command ["release-js/wrap_bare.sh"]}
:advanced {:source-paths ["src"]
:compiler {:output-to "target/advanced/datomish.js"
:optimizations :advanced
:source-map "target/advanced/datomish.js.map"
:pretty-print true
:recompile-dependents true
:parallel-build true
}}
:test {
:source-paths ["src" "test"]
:compiler {:output-to "target/test/datomish.js"
:output-dir "target/test"
:main datomish.test
:optimizations :none
:source-map true
:recompile-dependents true
:parallel-build true
:target :nodejs
}}
}
}
:cljsbuild {:builds
{
:release-node
{
:source-paths ["src-node" "src"]
:assert false
:compiler
{
:elide-asserts true
:hashbang false
:language-in :ecmascript5
:language-out :ecmascript5
:optimizations :advanced
:output-dir "release-node"
:output-to "release-node/datomish.bare.js"
:output-wrapper false
:parallel-build true
:pretty-print false
:target :nodejs
}
:notify-command ["release-node/wrap_bare.sh"]}
:release-browser
;; Release builds for use in Firefox must:
;; * Use :optimizations > :none, so that a single file is generated
;; without a need to import Closure's own libs.
;; * Be wrapped, so that a CommonJS module is produced.
;; * Have a preload script that defines what `println` does.
;;
;; There's no point in generating a source map -- it'll be wrong
;; due to wrapping.
{
:source-paths ["src-browser" "src"]
:assert false
:compiler
{
:elide-asserts true
:externs ["src-browser/externs.js"]
:language-in :ecmascript5
:language-out :ecmascript5
:optimizations :advanced
:output-dir "release-browser"
:output-to "release-browser/datomish.bare.js"
:output-wrapper false
:parallel-build true
:preloads [datomish.preload]
:pretty-print true
:pseudo-names true
:static-fns true
}
:notify-command ["release-browser/wrap_bare.sh"]}
:advanced
{:source-paths ["src-node" "src"]
:compiler
{
:language-in :ecmascript5
:language-out :ecmascript5
:output-dir "target/advanced"
:output-to "target/advanced/datomish.js"
:optimizations :advanced
:parallel-build true
:pretty-print true
:source-map "target/advanced/datomish.js.map"
:target :nodejs
}}
:test
{
:source-paths ["src-node" "src" "test"]
:compiler
{
:language-in :ecmascript5
:language-out :ecmascript5
:main datomish.test
:optimizations :none
:output-dir "target/test"
:output-to "target/test/datomish.js"
:parallel-build true
:source-map true
:target :nodejs
}}
}}
:profiles {:dev {:dependencies [[cljsbuild "1.1.3"]
[tempfile "0.2.0"]
@ -59,7 +111,26 @@
:doo {:build "test"}
:clean-targets ^{:protect false} ["target"
"release-js/datomish.bare.js"
"release-js/datomish.js"]
:clean-targets ^{:protect false}
[
"target"
"release-node/cljs/"
"release-node/cljs_promises/"
"release-node/clojure/"
"release-node/datascript/"
"release-node/datomish/"
"release-node/honeysql/"
"release-node/taoensso/"
"release-node/datomish.bare.js"
"release-node/datomish.js"
"release-browser/cljs/"
"release-browser/cljs_promises/"
"release-browser/clojure/"
"release-browser/datascript/"
"release-browser/datomish/"
"release-browser/honeysql/"
"release-browser/taoensso/"
"release-browser/datomish.bare.js"
"release-browser/datomish.js"
]
)

0
release-js/README.md → release-browser/README.md
View file

0
release-js/test_include_node.js → release-browser/test_include_node.js
View file

7
release-browser/wrap_bare.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/sh
set -e
(cat release-browser/wrapper.prefix; cat release-browser/datomish.bare.js; cat release-browser/wrapper.suffix) > release-browser/datomish.js
echo "Packed release-browser/datomish.js"

34
release-browser/wrapper.prefix Normal file
View file

@ -0,0 +1,34 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
// Datomish 0.1.0-SNAPSHOT
(function (definition) {
// This file will function properly as a <script> tag, or a module
// using CommonJS and NodeJS or RequireJS module formats.
// Wrapper gratefully adapted from:
// https://github.com/kriskowal/q/blob/v1/q.js
// https://github.com/swannodette/mori/blob/master/support/wrapper.js
// https://github.com/tonsky/datascript/blob/master/release-js/wrapper.js
// CommonJS
if (typeof exports === "object") {
module.exports = definition();
// RequireJS
} else if (typeof define === "function" && define.amd) {
define(definition);
// <script>
} else {
datomish = definition();
}
})(function () {
return function () {
// Monkeypatch setTimeout so that the Closure Compiler
// output can use it in a Sandbox context.
var { setTimeout } = require("sdk/timers");
this.setTimeout = setTimeout;

0
release-js/wrapper.suffix → release-browser/wrapper.suffix
View file

7
release-js/wrap_bare.sh
View file

@ -1,7 +0,0 @@
#!/bin/sh
set -e
(cat release-js/wrapper.prefix; cat release-js/datomish.bare.js; cat release-js/wrapper.suffix) > release-js/datomish.js
echo "Packed release-js/datomish.js"

5
release-node/README.md Normal file
View file

@ -0,0 +1,5 @@
# Datomish
Datomish is a persistent, embedded knowledge base. It's written in ClojureScript, and draws heavily on [DataScript](https://github.com/tonsky/datascript) and [Datomic](http://datomic.com).
For more info, check out the [project page](https://github.com/mozila/datomish).

2
release-node/test_include_node.js Normal file
View file

@ -0,0 +1,2 @@
var d = require('./datomish');
console.log(d.q("[:find ?e ?v :where [?e \"name\" ?v] {:x :y}]"));

7
release-node/wrap_bare.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/sh
set -e
(cat release-node/wrapper.prefix && cat release-node/datomish.bare.js && cat release-node/wrapper.suffix) > release-node/datomish.js
echo "Packed release-node/datomish.js"

0
release-js/wrapper.prefix → release-node/wrapper.prefix
View file

6
release-node/wrapper.suffix Normal file
View file

@ -0,0 +1,6 @@
;return this.datomish.js;
}.call({});
});

13
src-browser/datomish/core.cljs Normal file
View file

@ -0,0 +1,13 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.core
(:require
[honeysql.format :as sql]
[datomish.db :as db]
[datomish.db-factory :as db-factory]
[datomish.js-sqlite :as js-sqlite]
[datomish.sqlite :as sqlite]
[datomish.transact :as transact]))

20
src-browser/datomish/js_sqlite.cljs Normal file
View file

@ -0,0 +1,20 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js-sqlite
(:require
[datomish.sqlite :as s]
[datomish.js-util :refer [is-node?]]
[datomish.sqlitejsm-sqlite :as sqlitejsm-sqlite]))
(def open sqlitejsm-sqlite/open)
(extend-protocol s/ISQLiteConnectionFactory
string
(<sqlite-connection [path]
(open path))
object
(<sqlite-connection [tempfile]
(open (.-name tempfile))))

8
src-browser/datomish/preload.cljs Normal file
View file

@ -0,0 +1,8 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.preload)
(enable-console-print!)
(println "Console printing enabled.")

57
src-browser/datomish/sqlitejsm_sqlite.cljs Normal file
View file

@ -0,0 +1,57 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.sqlitejsm-sqlite
(:require
[cljs-promises.async]
[datomish.sqlite :as s]))
(def sqlite (.import (aget (js/require "chrome") "Cu") "resource://gre/modules/Sqlite.jsm"))
(println "sqlite is" (pr-str sqlite))
;; mozIStorageRow instances expose two methods: getResultByIndex and getResultByName.
;; Our code expects to treat rows as associative containers, from keyword to value.
;; So we implement ILookup (which has a different signature for ClojureScript than
;; Clojure!), hope that we handle nil/NULL correctly, and switch between integers
;; and keywords.
(deftype
StorageRow
[row]
ILookup
(-lookup [o k]
(-lookup o k nil))
(-lookup [o k not-found]
(or (if (integer? k)
(.getResultByIndex row k)
(.getResultByName row (clj->js (name k))))
not-found)))
(defrecord SQLite3Connection [db]
s/ISQLiteConnection
(-execute!
[db sql bindings]
(cljs-promises.async/pair-port
(.execute (.-db db) sql (or (clj->js bindings) #js []))))
(-each
[db sql bindings row-cb]
(let [cb (fn [row]
(row-cb (StorageRow. row)))]
(cljs-promises.async/pair-port
(.execute (.-db db) sql (or (clj->js bindings) #js []) (when row-cb cb)))))
(close
[db]
(cljs-promises.async/pair-port
(.close (.-db db)))))
(defn open
[path & {:keys [mode] :or {mode 6}}]
(cljs-promises.async/pair-port
(->
(.openConnection (aget sqlite "Sqlite") (clj->js {:path path :sharedMemoryCache false}))
(.then ->SQLite3Connection))))

33
src-browser/externs.js Normal file
View file

@ -0,0 +1,33 @@
var SqliteStatic = {};
/**
* @param {Object} options
* @return {Promise.<Sqlite>}
*/
SqliteStatic.openConnection = function (options) {}
var Sqlite = {}
/**
* @param {string} sql
* @param {Array} bindings
* @return {Promise}
*/
Sqlite.execute = function (sql, bindings) {}
/**
* @return {Promise}
*/
Sqlite.close = function() {}
var StorageRow = {};
/**
* @param {string} index
*/
StorageRow.getResultByIndex = function (index) {}
/**
* @param {string} name
*/
StorageRow.getResultByName = function (name) {}

0
src/datomish/core.cljs → src-node/datomish/core.cljs
View file

20
src-node/datomish/js_sqlite.cljs Normal file
View file

@ -0,0 +1,20 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js-sqlite
(:require
[datomish.sqlite :as s]
[datomish.js-util :refer [is-node?]]
[datomish.promise-sqlite :as promise-sqlite]))
(def open promise-sqlite/open)
(extend-protocol s/ISQLiteConnectionFactory
string
(<sqlite-connection [path]
(open path))
object ;; TODO: narrow this to the result of node-tempfile/tempfile.
(<sqlite-connection [tempfile]
(open (.-name tempfile))))

0
src/datomish/node_tempfile.cljs → src-node/datomish/node_tempfile.cljs
View file

0
src/datomish/node_tempfile_macros.cljc → src-node/datomish/node_tempfile_macros.cljc
View file

9
src/datomish/promise_sqlite.cljs → src-node/datomish/promise_sqlite.cljs
View file

@ -35,12 +35,3 @@
(->
(.open sqlite.DB path (clj->js {:mode mode}))
(.then ->SQLite3Connection))))
(extend-protocol s/ISQLiteConnectionFactory
string
(<sqlite-connection [path]
(open path))
object ;; TODO: narrow this to the result of node-tempfile/tempfile.
(<sqlite-connection [tempfile]
(open (.-name tempfile))))

111
src/datomish/exec_repl.cljc
View file

@ -1,111 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.exec-repl
#?(:cljs
(:require-macros
[datomish.util :refer [while-let]]
[datomish.pair-chan :refer [go-pair <?]]
[cljs.core.async.macros :refer [go]]))
(:require
[datomish.db-factory]
[datomish.db :as db]
[datomish.sqlite :as s]
[datomish.sqlite-schema :as ss]
[datomish.query :as dq]
#?@(:clj
[[datomish.jdbc-sqlite]
[datomish.pair-chan :refer [go-pair <?]]
[datomish.util :refer [while-let]]
[clojure.core.async :refer [<!]]])
#?@(:cljs
[[datomish.promise-sqlite]
[datomish.pair-chan]
[datomish.util]
[cljs.core.async :refer [<!]]])))
#?(:clj
(defn pair-channel->lazy-seq
"Returns a blocking lazy sequence of items taken from the provided channel."
[channel]
(lazy-seq
(when-let [v (clojure.core.async/<!! channel)]
(if (second v)
(cons v nil)
(cons v (pair-channel->lazy-seq channel)))))))
#?(:clj
(defn run-to-pair-seq
"Given an open database, returns a lazy sequence of results.
When fully consumed, underlying resources will be released."
[db find]
(pair-channel->lazy-seq (db/<?run db find))))
#_
(defn xxopen []
(datomish.pair-chan/go-pair
(let [d (datomish.pair-chan/<? (s/<sqlite-connection "/tmp/import.sqlite"))]
(clojure.core.async/<!! (ss/<ensure-current-version d))
(def db d))))
;; With an open DB…
#_(run-to-pair-seq
db
'[:find ?page :in $ :where [?page :page/starred true ?t]])
;; In a Clojure REPL with no open DB…
#_(clojure.core.async/<!!
(datomish.exec-repl/<open-and-run-to-seq-promise
"/tmp/foo.sqlite"
'[:find ?page :in $ :where [?page :page/starred true ?t]]))
#_(require 'datomish.exec-repl)
#_(in-ns 'datomish.exec-repl)
#_
(go-pair
(let [connection (<? (s/<sqlite-connection "/tmp/bigport.db"))
d (<? (datomish.db-factory/<db-with-sqlite-connection connection))]
(def db d)))
#_
(go-pair
(println (count (first (time
(<! (db/<?q db
'[:find ?url ?title :in $ :where
[?page :page/visitAt ?v] [(> ?v 1438748166567751)] [?page :page/title ?title] [?page :page/url ?url] ] {})))))))
#_
(go-pair
(let [connection (<? (s/<sqlite-connection "/tmp/foo.sqlite"))
dd (<? (datomish.db-factory/<db-with-sqlite-connection connection))]
(def *db* dd)))
#_
(clojure.core.async/<!!
(go-pair
(let [now -1
forms (mapcat (fn [i]
(map (fn [j]
[:db/add i :x j true])
(range 1000 (* i 2000) i)))
(range 1 10))]
(println "Adding" (count forms) "forms")
(<? (transact/<transact! *db* forms nil now)))))
#_
(go-pair
(let [connection (<? (s/<sqlite-connection "/tmp/foo.sqlite"))
dd (<? (db/<with-sqlite-connection connection))]
(println
(count
(<? (db/<?q dd
'[:find ?e ?v :in $ :where
[?e :x ?v]
#_[(> ?v 1000)]] {}))))))
(dq/parse '[:find ?entity ?tx ?score
:in $ ?search
:where [(foobar $ :artist/name ?search) [[?entity _ ?tx ?score]]]])
(honeysql.core/format {:select [:?foo] :from [:foo] :where [:match :foo.x "Bar"]})

40
src/datomish/js.cljs
View file

@ -1,11 +1,47 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js
(:refer-clojure :exclude [])
(:require-macros
[datomish.pair-chan :refer [go-pair <?]])
(:require
[datomish.core :as d]
[cljs.reader]))
[cljs.core.async :as a :refer [take! <! >!]]
[cljs.reader]
[cljs-promises.core :refer [promise]]
[datomish.db :as db]
[datomish.db-factory :as db-factory]
[datomish.pair-chan]
[datomish.sqlite :as sqlite]
[datomish.js-sqlite :as js-sqlite]
[datomish.transact :as transact]))
(defn- take-pair-as-promise! [ch]
;; Just like take-as-promise!, but aware that it's handling a pair channel.
(promise
(fn [resolve reject]
(letfn [(split-pair [[v e]]
(if e
(reject e)
(resolve v)))]
(cljs.core.async/take! ch split-pair)))))
;; Public API.
(defn ^:export open [path]
;; Eventually, URI. For now, just a plain path (no file://).
(take-pair-as-promise!
(go-pair
(let [conn (<? (sqlite/<sqlite-connection path))
db (<? (db-factory/<db-with-sqlite-connection conn))]
(let [c (transact/connection-with-db db)]
(clj->js
{:conn c
:close (fn [] (db/close-db db))
:toString (fn [] (str "#<DB " path ">"))
:path path}))))))
(defn ^:export q [query & sources]
(let [query (cljs.reader/read-string query)]
(clj->js query)))

14
src/datomish/js_util.cljs Normal file
View file

@ -0,0 +1,14 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
(ns datomish.js-util)
(defn is-node? []
(try
(= "[object process]"
(.toString (aget js/global "process")))
(catch js/ReferenceError e
false)
(catch js/TypeError e
false)))

4
src/datomish/query/clauses.cljc
View file

@ -126,8 +126,8 @@
"Transform a DataScript Pattern instance into the parts needed
to build a SQL expression.
@arg cc A CC instance.
@arg pattern The pattern instance.
@param cc A CC instance.
@param pattern The pattern instance.
@return an augmented CC"
[cc pattern]
(when-not (instance? Pattern pattern)

4
src/datomish/sqlite.cljc
View file

@ -101,7 +101,9 @@
(defn get-user-version [db]
(go-pair
(let [row (first (<? (all-rows db ["PRAGMA user_version"])))]
(:user_version row))))
(or
(:user_version row)
0))))
(defn set-user-version [db version]
(execute! db [(str "PRAGMA user_version = " version)]))

2
test/datomish/db_test.cljc
View file

@ -22,7 +22,7 @@
[datomish.test-macros :refer [deftest-async deftest-db]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async deftest-db]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/places/import_test.cljc
View file

@ -21,7 +21,7 @@
[datomish.test-macros :refer [deftest-async]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/promise_sqlite_test.cljs
View file

@ -14,7 +14,7 @@
[cljs.test :refer-macros [is are deftest testing async]]
[datomish.pair-chan]
[datomish.sqlite :as s]
[datomish.promise-sqlite :as ps]))
[datomish.js-sqlite :as ps]))
(deftest-async test-all-rows
(with-tempfile [t (tempfile)]

2
test/datomish/query_test.cljc
View file

@ -16,7 +16,7 @@
[datomish.test-macros :refer [deftest-async]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/test/query.cljc
View file

@ -22,7 +22,7 @@
[tempfile.core :refer [tempfile with-tempfile]]
[clojure.test :as t :refer [is are deftest testing]]])
#?@(:cljs
[[datomish.promise-sqlite]
[[datomish.js-sqlite]
[datomish.test-macros :refer-macros [deftest-db]]
[honeysql.core :as sql :refer-macros [param]]
[datomish.node-tempfile :refer [tempfile]]

2
test/datomish/tofinoish_test.cljc
View file

@ -17,7 +17,7 @@
[datomish.test-macros :refer [deftest-async deftest-db]]
[clojure.test :as t :refer [is are deftest testing]]
[clojure.core.async :refer [go <! >!]]])
#?@(:cljs [[datomish.promise-sqlite]
#?@(:cljs [[datomish.js-sqlite]
[datomish.pair-chan]
[datomish.test-macros :refer-macros [deftest-async deftest-db]]
[datomish.node-tempfile :refer [tempfile]]