[sdks/android] Build Mentat Android SDK in TaskCluster; publish org.mozilla.mentat to nalexander's personal Bintray repo.

The automation parts were cribbed directly from
50add3e176.

The automation permissions were added in
https://bugzilla.mozilla.org/show_bug.cgi?id=1477311.

This uses a very rudimentary Gradle plugin, `rust-android-gradle`,
with custom fixes and extensions.  It works pretty well for what it
is!  See https://github.com/ncalexan/rust-android-gradle.

.taskcluster.yml

@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+version: 0
+allowPullRequests: public
+tasks:
+####################################################################################################
+# Task: Pull requests
+####################################################################################################
+  - provisionerId: '{{ taskcluster.docker.provisionerId }}'
+    workerType: '{{ taskcluster.docker.workerType }}'
+    extra:
+      github:
+        env: true
+        events:
+          - pull_request.opened
+          - pull_request.edited
+          - pull_request.synchronize
+          - pull_request.reopened
+          - push
+    scopes:
+      - "queue:create-task:aws-provisioner-v1/github-worker"
+      - "queue:scheduler-id:taskcluster-github"
+    payload:
+      maxRunTime: 3600
+      deadline: "{{ '2 hours' | $fromNow }}"
+      image: 'mozillamobile/mentat:1.1'
+      command:
+        - /bin/bash
+        - '--login'
+        - '-cx'
+        - >-
+          export TERM=dumb
+          && git fetch {{ event.head.repo.url }} {{ event.head.repo.branch }}
+          && git config advice.detachedHead false
+          && git checkout {{event.head.sha}}
+          && python automation/taskcluster/decision_task_pull_request.py
+      features:
+        taskclusterProxy: true
+    metadata:
+      name: Mentat Android SDK - Pull Request
+      description: Building and testing the Mentat Android SDK - triggered by a pull request.
+      owner: '{{ event.head.user.email }}'
+      source: '{{ event.head.repo.url }}'
+####################################################################################################
+# Task: Release
+####################################################################################################
+  - provisionerId: '{{ taskcluster.docker.provisionerId }}'
+    workerType: '{{ taskcluster.docker.workerType }}'
+    extra:
+      github:
+        events:
+          - release
+    scopes:
+      - "secrets:get:project/mentat/publish"
+    payload:
+      maxRunTime: 3600
+      deadline: "{{ '2 hours' | $fromNow }}"
+      image: 'mozillamobile/mentat:1.1'
+      command:
+        - /bin/bash
+        - '--login'
+        - '-cx'
+        - >-
+          export TERM=dumb
+          && git fetch origin --tags
+          && git config advice.detachedHead false
+          && git checkout {{ event.version }}
+          && python automation/taskcluster/release/fetch-bintray-api-key.py
+          && cd sdks/android/Mentat
+          && ./gradlew --no-daemon clean library:assembleRelease
+          && ./gradlew bintrayUpload --debug
+      features:
+        taskclusterProxy: true
+    metadata:
+      name: Mentat Android SDK - Release ({{ event.version }})
+      description: Building and publishing release versions.
+      owner: '{{ event.head.user.email }}'
+      source: '{{ event.head.repo.url }}'

CHANGELOG.md

@@ -0,0 +1,11 @@
+# 0.9 (2018-07-25)
+
+* sdks/android compiled against:
+  * Kotlin standard library 1.2.41
+
+* **API changes**:
+  * Mentat partitions now enforce their integrity, denying entids that aren't already known.
+
+* **sdks/android**: First version published to nalexander's personal bintray repository.
+* Various bugfixes and refactorings (see commits below for details)
+* [Commits](https://github.com/mozilla-mobile/android-components/compare/v0.8.1...v0.9)

Cargo.toml

@@ -12,7 +12,7 @@ authors = [
   "Thom Chiovoloni <tchiovoloni@mozilla.com>",
 ]
 name = "mentat"
-version = "0.8.1"
+version = "0.9.0"
 build = "build/version.rs"
 
 [features]

automation/docker/Dockerfile

@@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM mozillamobile/android-components:1.4
+
+MAINTAINER Nick Alexander "nalexander@mozilla.com"
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- Configuration -----------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+ENV ANDROID_NDK_VERSION "r17b"
+ENV PROJECT_REPOSITORY "https://github.com/mozilla/mentat.git"
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- System ------------------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+RUN apt-get update -qq
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- Android NDK (Android SDK comes from base `android-components` image) ----------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+RUN mkdir -p /build
+WORKDIR /build
+
+# ENV ANDROID_HOME /build/android-sdk
+# ENV ANDROID_SDK_HOME /build/android-sdk
+ENV ANDROID_NDK_HOME /build/android-ndk
+# ENV PATH ${PATH}:${ANDROID_NDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS}
+
+RUN curl -L https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip > ndk.zip \
+	&& unzip ndk.zip -d /build \
+	&& rm ndk.zip \
+  && mv /build/android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME}
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- Rust (cribbed from https://github.com/rust-lang-nursery/docker-rust/blob/ced83778ec6fea7f63091a484946f95eac0ee611/1.27.1/stretch/Dockerfile)
+#-- Rust after the Android NDK since Rust rolls forward more frequently.
+#----------------------------------------------------------------------------------------------------------------------
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.27.1
+
+RUN set -eux; \
+    rustArch='x86_64-unknown-linux-gnu'; rustupSha256='4d382e77fd6760282912d2d9beec5e260ec919efd3cb9bdb64fe1207e84b9d91'; \
+    url="https://static.rust-lang.org/rustup/archive/1.12.0/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --default-toolchain $RUST_VERSION; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version; \
+    rustup target add i686-linux-android; \
+    rustup target add arm-linux-androideabi; \
+    rustup target add aarch64-linux-android
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- Project -----------------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+RUN git clone $PROJECT_REPOSITORY
+
+WORKDIR /build/mentat
+
+# Temporary.
+RUN git fetch origin build-android-sdk && git checkout origin/build-android-sdk &&    git show-ref HEAD
+
+# Populate dependencies.
+RUN ./sdks/android/Mentat/gradlew --no-daemon -p sdks/android/Mentat tasks
+
+# Cache toolchains.
+RUN ./sdks/android/Mentat/gradlew --no-daemon -p sdks/android/Mentat generateToolchains
+
+# Build Rust.
+RUN ./sdks/android/Mentat/gradlew --no-daemon -p sdks/android/Mentat cargoBuild
+
+# Actually build.  In the future, we might also test and lint (to cache additional dependencies).
+RUN ./sdks/android/Mentat/gradlew --no-daemon -p sdks/android/Mentat assemble
+
+# Drop built Rust artifacts.
+RUN cargo clean
+
+#----------------------------------------------------------------------------------------------------------------------
+# -- Cleanup ----------------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+RUN apt-get clean

automation/taskcluster/decision_task_pull_request.py

@@ -0,0 +1,122 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import json
+import os
+import taskcluster
+import re
+import subprocess
+import sys
+
+
+"""
+Decision task for pull requests
+"""
+
+TASK_ID = os.environ.get('TASK_ID')
+REPO_URL = os.environ.get('GITHUB_HEAD_REPO_URL')
+BRANCH = os.environ.get('GITHUB_HEAD_BRANCH')
+COMMIT = os.environ.get('GITHUB_HEAD_SHA')
+
+def fetch_module_names():
+    process = subprocess.Popen(["./gradlew", "--no-daemon", "printModules"], stdout=subprocess.PIPE,
+                               cwd=os.path.join(os.getcwd(), "sdks", "android", "Mentat"))
+    (output, err) = process.communicate()
+    exit_code = process.wait()
+
+    if exit_code is not 0:
+        print "Gradle command returned error:", exit_code
+
+    return re.findall('module: (.*)', output, re.M)
+
+
+def schedule_task(queue, taskId, task):
+    print "TASK", taskId
+    print json.dumps(task, indent=4, separators=(',', ': '))
+
+    result = queue.createTask(taskId, task)
+    print "RESULT", taskId
+    print json.dumps(result, indent=4, separators=(',', ': '))
+
+
+def create_task(name, description, command):
+    created = datetime.datetime.now()
+    expires = taskcluster.fromNow('1 year')
+    deadline = taskcluster.fromNow('1 day')
+
+    return {
+        "workerType": 'github-worker',
+        "taskGroupId": TASK_ID,
+        "expires": taskcluster.stringDate(expires),
+        "retries": 5,
+        "created": taskcluster.stringDate(created),
+        "tags": {},
+        "priority": "lowest",
+        "schedulerId": "taskcluster-github",
+        "deadline": taskcluster.stringDate(deadline),
+        "dependencies": [ TASK_ID ],
+        "routes": [],
+        "scopes": [],
+        "requires": "all-completed",
+        "payload": {
+            "features": {},
+            "maxRunTime": 7200,
+            "image": "mozillamobile/mentat:1.1",
+            "command": [
+                "/bin/bash",
+                "--login",
+                "-cx",
+                "export TERM=dumb && git fetch %s %s && git config advice.detachedHead false && git checkout %s && cd sdks/android/Mentat && ./gradlew --no-daemon clean %s" % (REPO_URL, BRANCH, COMMIT, command)
+            ],
+            "artifacts": {},
+            "deadline": taskcluster.stringDate(deadline)
+        },
+        "provisionerId": "aws-provisioner-v1",
+        "metadata": {
+            "name": name,
+            "description": description,
+            "owner": "nalexander@mozilla.com",
+            "source": "https://github.com/mozilla/mentat"
+        }
+    }
+
+
+def create_module_task(module):
+    return create_task(
+        name='Mentat Android SDK - Module ' + module,
+        description='Building and testing module ' + module,
+        command=" ".join(map(lambda x: module + ":" + x, ['assemble', 'test', 'lint'])))
+
+
+# def create_detekt_task():
+#     return create_task(
+#         name='Android Components - detekt',
+#         description='Running detekt over all modules',
+#         command='detektCheck')
+
+
+# def create_ktlint_task():
+#     return create_task(
+#         name='Android Components - ktlint',
+#         description='Running ktlint over all modules',
+#         command='ktlint')
+
+
+if __name__ == "__main__":
+    queue = taskcluster.Queue({ 'baseUrl': 'http://taskcluster/queue/v1' })
+
+    modules = fetch_module_names()
+
+    if len(modules) == 0:
+        print "Could not get module names from gradle"
+        sys.exit(2)
+
+    for module in modules:
+        task = create_module_task(module)
+        task_id = taskcluster.slugId()
+        schedule_task(queue, task_id, task)
+
+    # schedule_task(queue, taskcluster.slugId(), create_detekt_task())
+    # schedule_task(queue, taskcluster.slugId(), create_ktlint_task())

automation/taskcluster/release/fetch-bintray-api-key.py

@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import taskcluster
+
+SECRET_NAME = 'project/mentat/publish'
+TASKCLUSTER_BASE_URL = 'http://taskcluster/secrets/v1'
+
+def fetch_publish_secrets(secret_name):
+    """Fetch and return secrets from taskcluster's secret service"""
+    secrets = taskcluster.Secrets({'baseUrl': TASKCLUSTER_BASE_URL})
+    return secrets.get(secret_name)
+
+def main():
+    """Fetch the bintray user and api key from taskcluster's secret service
+    and save it to local.properties in the project root directory.
+    """
+    data = fetch_publish_secrets(SECRET_NAME)
+
+    properties_file_path = os.path.join(os.path.dirname(__file__), '../../../sdks/android/Mentat/local.properties')
+    with open(properties_file_path, 'w') as properties_file:
+        properties_file.write("bintray.user=%s\n" % data['secret']['bintray_user'])
+        properties_file.write("bintray.apikey=%s\n" % data['secret']['bintray_apikey'])
+
+if __name__ == "__main__":
+    main()

sdks/android/Mentat/build.gradle

@@ -4,27 +4,32 @@ buildscript {
     ext.kotlin_version = '1.2.41'
 
     ext.library = [
-            version: '0.3.1'
+        version: '0.9.0'
     ]
 
     ext.build = [
         compileSdkVersion: 27,
         targetSdkVersion: 27,
-            minSdkVersion: 19
+        minSdkVersion: 21, // So that we can publish for aarch64.
     ]
 
     repositories {
         jcenter()
         google()
+        maven {
+            url "https://plugins.gradle.org/m2/"
         }
+    }
+
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.1.2'
+        classpath 'com.android.tools.build:gradle:3.1.3'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
 
         // Publish.
         classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7.3'
         classpath 'com.github.dcendents:android-maven-gradle-plugin:2.1'
 
+        classpath 'gradle.plugin.org.mozilla.rust-android-gradle:plugin:0.0.4'
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
     }
@@ -32,11 +37,19 @@ buildscript {
 
 allprojects {
     repositories {
-        jcenter()
         google()
+        jcenter()
     }
 }
 
 task clean(type: Delete) {
     delete rootProject.buildDir
 }
+
+task printModules {
+    doLast {
+        subprojects.each { p ->
+            println "module: " + p.path
+        }
+    }
+}

sdks/android/Mentat/gradle.properties

@@ -10,6 +10,7 @@
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
 org.gradle.jvmargs=-Xmx1536m
+org.gradle.configureondemand=false
 
 # When configured, Gradle will run in incubating parallel mode.
 # This option should only be used with decoupled projects. More details, visit
@@ -17,10 +18,11 @@ org.gradle.jvmargs=-Xmx1536m
 # org.gradle.parallel=true
 
 libGroupId=org.mozilla.mentat
-libRepositoryName=Mentat
+libRepositoryName=mentat
 libProjectName=mentat
 libProjectDescription=A persistent, relational store inspired by Datomic and DataScript.
 libUrl=https://github.com/mozilla/mentat
 libVcsUrl=https://github.com/mozilla/mentat.git
 
-libLicense=MPL-2.0
+libLicense 'Apache-2.0'
+libLicenseUrl 'http://www.apache.org/licenses/LICENSE-2.0.txt'

sdks/android/Mentat/gradle/wrapper/gradle-wrapper.properties

@@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-all.zip

sdks/android/Mentat/library/build.gradle

@@ -1,4 +1,6 @@
 apply plugin: 'com.android.library'
+apply plugin: 'org.mozilla.rust-android-gradle.rust-android'
+
 apply plugin: 'kotlin-android'
 apply plugin: 'com.jfrog.bintray'
 // Simply applying this plugin gets bintray to publish a pom file.
@@ -33,6 +35,18 @@ android {
     }
 }
 
+cargo {
+    module = '../../../../ffi'
+    targetDirectory = '../../../../target'
+    targetInclude = 'libmentat_ffi.so'
+
+    targets = [
+        'x86',
+        'arm',
+        'arm64',
+    ]
+}
+
 dependencies {
     androidTestImplementation 'com.android.support:support-annotations:27.1.1'
     androidTestImplementation 'com.android.support.test:runner:1.0.2'
@@ -41,54 +55,27 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
     implementation 'net.java.dev.jna:jna:4.5.1'
 }
+
 repositories {
     mavenCentral()
 }
 
-// Publishing to jcenter/bintray.
-def libGroupId = properties.libGroupId
-def libRepoName = properties.libRepositoryName
-def libProjectName = properties.libProjectName
-def libProjectDescription = properties.libProjectDescription
-def libUrl = properties.libUrl
-def libVcsUrl = properties.libVcsUrl
-def libLicense = properties.libLicense
-
-Properties localProperties = null
-if (project.rootProject.file('local.properties').canRead()) {
-    localProperties = new Properties()
-    localProperties.load(project.rootProject.file('local.properties').newDataInputStream())
-}
-
-version = rootProject.ext.library['version']
-
-task sourcesJar(type: Jar) {
-    from android.sourceSets.main.java.srcDirs
-    classifier = 'sources'
-}
-
-artifacts {
-    archives sourcesJar
-}
-
-
-group = libGroupId
-archivesBaseName = libProjectName
-
-bintray {
-    user = localProperties != null ? localProperties.getProperty("bintray.user") : ""
-    key = localProperties != null ? localProperties.getProperty("bintray.apikey") : ""
-
-    configurations = ['archives']
-    pkg {
-        repo = libRepoName
-        name = libProjectName
-        userOrg = "grisha" // Temporary org name until package is on jcenter. Issue #725.
-        desc = libProjectDescription
-        websiteUrl = libUrl
-        vcsUrl = libVcsUrl
-        licenses = [libLicense]
-        publish = true
-        publicDownloadNumbers = true
+afterEvaluate {
+    // The `cargoBuild` tasks isn't available until after evaluation.
+    android.libraryVariants.all { variant ->
+        def productFlavor = ""
+        variant.productFlavors.each {
+            productFlavor += "${it.name.capitalize()}"
+        }
+        def buildType = "${variant.buildType.name.capitalize()}"
+        tasks["generate${productFlavor}${buildType}Assets"].dependsOn(tasks["cargoBuild"])
     }
 }
+
+archivesBaseName = 'mentat'
+
+apply from: '../publish.gradle'
+ext.configurePublish(
+        'org.mozilla.mentat',
+        'mentat',
+        'A persistent, embedded knowledge base.')

sdks/android/Mentat/library/src/main/jniLibs/arm64/libmentat_ffi.so

@@ -1 +0,0 @@
-../../../../../../../../target/aarch64-linux-android/release/libmentat_ffi.so

sdks/android/Mentat/library/src/main/jniLibs/armeabi/libmentat_ffi.so

@@ -1 +0,0 @@
-../../../../../../../../target/armv7-linux-androideabi/release/libmentat_ffi.so

sdks/android/Mentat/library/src/main/jniLibs/x86/libmentat_ffi.so

@@ -1 +0,0 @@
-../../../../../../../../target/i686-linux-android/release/libmentat_ffi.so

sdks/android/Mentat/publish.gradle

@@ -0,0 +1,99 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+def libRepoName = properties.libRepositoryName
+def libUrl = properties.libUrl
+def libVcsUrl = properties.libVcsUrl
+def libLicense = properties.libLicense
+def libLicenseUrl = properties.libLicenseUrl
+
+ext.configurePublish = { groupIdArg, artifactIdArg, descriptionArg ->
+    apply plugin: 'com.github.dcendents.android-maven'
+
+    group = groupIdArg
+
+    install {
+        repositories.mavenInstaller {
+            pom {
+                project {
+                    packaging 'aar'
+                    groupId groupIdArg
+                    artifactId artifactIdArg
+
+                    name libRepoName
+                    description descriptionArg
+                    url libUrl
+
+                    licenses {
+                        license {
+                            name libLicense
+                            url libLicenseUrl
+                        }
+                    }
+
+                    developers {
+                        developer {
+                            id 'nalexander'
+                            name 'Nick Alexander'
+                            email 'nalexander@mozilla.com'
+                        }
+                    }
+
+                    scm {
+                        connection libVcsUrl
+                        developerConnection libVcsUrl
+                        url libUrl
+                    }
+                }
+            }
+        }
+    }
+
+    apply plugin: 'com.jfrog.bintray'
+
+    version = rootProject.ext.library['version']
+
+    task sourcesJar(type: Jar) {
+        from android.sourceSets.main.java.srcDirs
+        classifier = 'sources'
+    }
+
+    task javadoc(type: Javadoc) {
+        source = android.sourceSets.main.java.srcDirs
+        classpath += project.files(android.getBootClasspath().join(File.pathSeparator))
+    }
+
+    task javadocJar(type: Jar, dependsOn: javadoc) {
+        classifier = 'javadoc'
+        from javadoc.destinationDir
+    }
+
+    artifacts {
+        //archives javadocJar
+        archives sourcesJar
+    }
+
+    Properties localProperties = null;
+    if (project.rootProject.file('local.properties').canRead()) {
+        localProperties = new Properties()
+        localProperties.load(project.rootProject.file('local.properties').newDataInputStream())
+    }
+
+    bintray {
+        user = localProperties != null ? localProperties.getProperty("bintray.user") : ""
+        key = localProperties != null ? localProperties.getProperty("bintray.apikey") : ""
+
+        configurations = ['archives']
+        pkg {
+            repo = libRepoName
+            name = artifactIdArg
+            desc = descriptionArg
+            websiteUrl = libUrl
+            vcsUrl = libVcsUrl
+            licenses = [libLicense]
+            publish = true
+            publicDownloadNumbers = true
+        }
+    }
+}