From d2000eed3ad9efcdeebeef78210169e517c5c43c Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 19 Sep 2023 16:37:58 -0400 Subject: [PATCH] WIP --- .github/TODO | 65 +++++ .github/workflows/main.yml | 21 +- .github/workflows/nix-github-actions.yml | 21 -- NOTES | 73 ++++++ _flake.nix | 51 ++++ branch_deployments.yml_ | 294 +++++++++++++++++++++++ main.yml | 52 ++++ 7 files changed, 552 insertions(+), 25 deletions(-) create mode 100644 .github/TODO delete mode 100644 .github/workflows/nix-github-actions.yml create mode 100644 NOTES create mode 100644 _flake.nix create mode 100644 branch_deployments.yml_ create mode 100644 main.yml diff --git a/.github/TODO b/.github/TODO new file mode 100644 index 0000000..f4fd5d7 --- /dev/null +++ b/.github/TODO @@ -0,0 +1,65 @@ + steps: + - uses: actions/checkout@v3 + - uses: uraimo/run-on-arch-action@v2 + if: ${{ matrix.arch }} != x86_64 + name: Build artifact + id: build + with: + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + + # Not required, but speeds up builds + githubToken: ${{ github.token }} + + # Set an output parameter `uname` for use in subsequent steps + run: | + uname -a + echo ::set-output name=uname::$(uname -a) + + # Create an artifacts directory + setup: | + mkdir -p "${PWD}/artifacts" + + # Mount the artifacts directory as /artifacts in the container + dockerRunArgs: | + --volume "${PWD}/artifacts:/artifacts" + + # Pass some environment variables to the container + env: | # YAML, but pipe character is necessary + artifact_name: git-${{ matrix.distro }}_${{ matrix.arch }} + + # The shell to run commands with in the container + shell: /bin/sh + + # Install some dependencies in the container. This speeds up builds if + # you are also using githubToken. Any dependencies installed here will + # be part of the container image that gets cached, so subsequent + # builds don't have to re-install them. The image layer is cached + # publicly in your project's package repository, so it is vital that + # no secrets are present in the container state or logs. + install: | + case "${{ matrix.distro }}" in + ubuntu*|jessie|stretch|buster|bullseye) + apt-get update -q -y + apt-get install -q -y git + ;; + fedora*) + dnf -y update + dnf -y install git which + ;; + alpine*) + apk update + apk add git + ;; + esac + + # Produce a binary artifact and place it in the mounted volume + run: | + cp $(which git) "/artifacts/${artifact_name}" + echo "Produced artifact at /artifacts/${artifact_name}" + + - name: Show the artifact + # Items placed in /artifacts in the container will be in + # ${PWD}/artifacts on the host. + run: | + ls -al "${PWD}/artifacts" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48c2a98..a51009c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,9 +9,9 @@ permissions: contents: read issues: write -# concurrency: -# group: ${{ github.ref }} -# cancel-in-progress: true +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true env: GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} @@ -19,14 +19,27 @@ env: GITHUB_SHA: ${{ github.sha }} jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + echo "matrix=$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" nix-build: needs: nix-matrix + name: Build on ${{ matrix.attr }} runs-on: ${{ matrix.os }} strategy: matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} steps: - - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v22 + - uses: actions/checkout@v3 - run: nix build -L ".#${{ matrix.attr }}" # runs-on: ubuntu-latest diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml deleted file mode 100644 index f260d5f..0000000 --- a/.github/workflows/nix-github-actions.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Nix Flake actions - -on: - pull_request: - types: [opened, reopened, synchronize] - push: - branches: [main, master] - -jobs: - nix-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v22 - - id: set-matrix - name: Generate Nix Matrix - run: | - set -Eeu - echo "matrix=$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" diff --git a/NOTES b/NOTES new file mode 100644 index 0000000..7a93863 --- /dev/null +++ b/NOTES @@ -0,0 +1,73 @@ + + platforms = [ + "i686-cygwin" + "x86_64-cygwin" + "x86_64-darwin" + "i686-darwin" + "aarch64-darwin" + "armv7a-darwin" + "i686-freebsd13" + "x86_64-freebsd13" + "aarch64-genode" + "i686-genode" + "x86_64-genode" + "x86_64-solaris" + "js-ghcjs" + "aarch64-linux" + "armv5tel-linux" + "armv6l-linux" + "armv7a-linux" + "armv7l-linux" + "i686-linux" + "m68k-linux" + "microblaze-linux" + "microblazeel-linux" + "mipsel-linux" + "mips64el-linux" + "powerpc64-linux" + "powerpc64le-linux" + "riscv32-linux" + "riscv64-linux" + "s390-linux" + "s390x-linux" + "x86_64-linux" + "mmix-mmixware" + "aarch64-netbsd" + "armv6l-netbsd" + "armv7a-netbsd" + "armv7l-netbsd" + "i686-netbsd" + "m68k-netbsd" + "mipsel-netbsd" + "powerpc-netbsd" + "riscv32-netbsd" + "riscv64-netbsd" + "x86_64-netbsd" + "aarch64_be-none" + "aarch64-none" + "arm-none" + "armv6l-none" + "avr-none" + "i686-none" + "microblaze-none" + "microblazeel-none" + "msp430-none" + "or1k-none" + "m68k-none" + "powerpc-none" + "powerpcle-none" + "riscv32-none" + "riscv64-none" + "rx-none" + "s390-none" + "s390x-none" + "vc4-none" + "x86_64-none" + "i686-openbsd" + "x86_64-openbsd" + "x86_64-redox" + "wasm64-wasi" + "wasm32-wasi" + "x86_64-windows" + "i686-windows" + ]; \ No newline at end of file diff --git a/_flake.nix b/_flake.nix new file mode 100644 index 0000000..812b7ae --- /dev/null +++ b/_flake.nix @@ -0,0 +1,51 @@ + # Tests run by 'nix flake check' and by Hydra. + checks = forAllSystems + (system: + with pkgs.${system}; + + { + inherit (self.packages.${system}) hello; + + # Additional tests, if applicable. + test = stdenv.mkDerivation { + pname = "hello-test"; + inherit version; + + buildInputs = [ hello ]; + + dontUnpack = true; + + buildPhase = '' + echo 'running some integration tests' + [[ $(hello) = 'Hello Nixers!' ]] + ''; + + installPhase = "mkdir -p $out"; + }; + } + + // lib.optionalAttrs stdenv.isLinux { + # A VM test of the NixOS module. + vmTest = + with import (nixpkgs + "/nixos/lib/testing-python.nix") { + inherit system; + }; + + makeTest { + nodes = { + client = { ... }: { + imports = [ self.nixosModules.hello ]; + }; + }; + + testScript = + '' + start_all() + client.wait_for_unit("multi-user.target") + client.succeed("hello") + ''; + }; + } + ); + + }; diff --git a/branch_deployments.yml_ b/branch_deployments.yml_ new file mode 100644 index 0000000..8ef908c --- /dev/null +++ b/branch_deployments.yml_ @@ -0,0 +1,294 @@ +name: Deploy a development branch PR using Dagster Hybrid Cloud + +# on: +# pull_request: +# types: [opened, synchronize, reopened] + +concurrency: + # Cancel in-progress runs on same branch + group: ${{ github.ref }} + cancel-in-progress: true + +env: + AWS_PROFILE: "klar-neutral" + AWS_ACCOUNT_ID: "739282534308" + AWS_REGION: "us-east-2" + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DAGSTER_CLOUD_ORGANIZATION_ID: "klar" + DAGSTER_CLOUD_URL: "https://klar.dagster.cloud" + DOCKER_CONFIG: $GITHUB_WORKSPACE + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SHA: ${{ github.sha }} + PIP_ROOT_USER_ACTION: ignore + GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} # For PRs, use the head branch name, for other commits, use the branch name itself + +jobs: + setup: + name: Parse Dagster Hybrid Cloud configuration file + runs-on: [self-hosted, default] + outputs: + locations: ${{ steps.load_workspace_file.outputs.build_info }} + secrets_set: ${{ steps.load_workspace_file.outputs.secrets_set }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + - run: | + git fetch origin ${{ github.event.repository.default_branch }}:${{ github.event.repository.default_branch }} + git fetch origin ${{ env.GIT_BRANCH_NAME }}:${{ env.GIT_BRANCH_NAME }} + + - name: Configure PIP to use AWS/CodeArtifact + run: | + CODEARTIFACT_AUTH_TOKEN=$(aws --profile klar-neutral codeartifact get-authorization-token --domain klar-neutral --query authorizationToken --output text) + pip config set global.index-url "https://aws:${CODEARTIFACT_AUTH_TOKEN}@klar-neutral-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-2.amazonaws.com/pypi/klar/simple/" + + - name: Install Virtual Environment for Python + shell: bash + run: python3 -m pip install virtualenv + + - name: Setup Virtual Environment for Python + shell: bash + run: virtualenv /tmp/venv + + - name: Activate Virtual Environment for Python + shell: bash + run: | + . /tmp/venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV + + - name: Display Python version + shell: bash + run: | + which python + python -c "import sys; print(sys.version)" + + - name: Upgrade pip + shell: bash + run: | + python -m pip install --upgrade pip + + - name: Install required Python dependencies + shell: bash + run: | + pip install --upgrade oyaml sh GitPython + + - name: Create build_info with only the changed code locations + id: load_workspace_file + shell: bash + env: + CHANGES_ONLY: yes + DEFAULT_BRANCH_NAME: ${{ github.event.repository.default_branch }} + run: | + python ${GITHUB_WORKSPACE}/.github/workflows/parse_workspace.py dagster_cloud.yaml ${{ env.GIT_BRANCH_NAME }} >> $GITHUB_OUTPUT + + build: + name: Dagster Hybrid Deploy + runs-on: [self-hosted, default] + needs: setup + if: ${{ needs.setup.outputs.locations != '[]' }} + permissions: write-all + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.setup.outputs.locations) }} + python-version: ["3.10"] + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} + lfs: true +# submodules: recursive + + - name: Copy common utils framework into this code location + shell: bash + working-directory: ${{ matrix.location.directory }} + run: | + [ ! -d utils ] && mkdir utils + (cd ${GITHUB_WORKSPACE} && tar -cf - utils) | tar -xvf - + + - name: Set deps-cache-from + # Don't use cached deps if instructed + shell: bash + if: ${{ inputs.force_rebuild_deps != 'true' }} + # For PR commits, use the target branch name as the cache-tag, for other commits, use the branch name itself + run: echo "FLAG_DEPS_CACHE_FROM=--deps-cache-from=${{ github.repository }}/${{ matrix.location.name }}/${{ github.base_ref && github.base_ref || github.ref_name }}" >> $GITHUB_ENV + + - name: Set deps-cache-to + # Only write to the cache-tag for non PR commits so PR commits don't upgrade dependency versions + shell: bash + if: ${{ github.base_ref == '' }} + run: echo "FLAG_DEPS_CACHE_TO=--deps-cache-to=${{ github.repository }}/${{ matrix.location.name }}/${{ github.ref_name }}" >> $GITHUB_ENV + + - name: Configure PIP to use AWS/CodeArtifact + run: | + export CODEARTIFACT_AUTH_TOKEN=$(aws --profile klar-neutral codeartifact get-authorization-token --domain klar-neutral --query authorizationToken --output text) + echo "PIP_INDEX_URL=https://aws:${CODEARTIFACT_AUTH_TOKEN}@klar-neutral-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-2.amazonaws.com/pypi/klar/simple/" >> $GITHUB_ENV + pip config set global.index-url "https://aws:${CODEARTIFACT_AUTH_TOKEN}@klar-neutral-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-2.amazonaws.com/pypi/klar/simple/" + + - name: Login to AWS/ECR + shell: bash + run: | + aws ecr get-login-password --region ${{ env.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com + + - name: Create ECR repository if it doesn't exist + id: create-repository + continue-on-error: true + shell: bash + run: | + output=$(aws ecr describe-repositories --repository-names "${{ matrix.location.name }}" 2>&1) + if [ $? -ne 0 ]; then + if echo "${output}" | grep -q RepositoryNotFoundException; then + aws ecr create-repository --repository-name "${{ matrix.location.name }}" + else + >&2 echo "${output}" + fi + fi + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install Wheel, SetupTools, Virtual Environment, and PIPx + shell: bash + run: | + python3 -m pip install wheel setuptools virtualenv pipx + + - name: Activate Virtual Environment for code location + shell: bash + working-directory: ${{ matrix.location.directory }} + run: | + virtualenv venv-${{ matrix.location.name }} + source ./venv-${{ matrix.location.name }}/bin/activate + echo PATH=$PATH >> $GITHUB_ENV + + - name: Display Python version + shell: bash + run: | + which python + python -c "import sys; print(sys.version)" + + - name: Upgrade pip + shell: bash + run: | + python -m pip install --upgrade pip + + - name: Install more Python dependencies + shell: bash + run: | + pip install coverage pytest pytest-json-report pytest-md-report pytest-annotate pyannotate ruff markupsafe==2.0.1 lintly + pip install oyaml sh GitPython + + - name: Install code location requirements + working-directory: ${{ matrix.location.directory }} + shell: bash + run: | + if [ -d utils ]; then + cp requirements.txt /tmp + sed -i -e "s|/opt/dagster/app|$(pwd)|" requirements.txt + pip install -e utils + mv /tmp/requirements.txt . + fi + pip install -e ".[tests]" + + - name: Run tests and collect coverage + working-directory: ${{ matrix.location.directory }} + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python -m coverage run -m pytest *.py --doctest-modules --ignore=${{ matrix.location.name }} \ + --md-report --md-report-output="/tmp/${{ matrix.location.name }}-code-coverage-results.md" + python -m coverage html --data-file=.coverage --directory=/tmp/${{ matrix.location.name }} --title=${{ matrix.location.name }} --rcfile=${GITHUB_WORKSPACE}/.coveragerc + python -m coverage json --data-file=.coverage --o coverage.json --rcfile=${GITHUB_WORKSPACE}/.coveragerc + python ${GITHUB_WORKSPACE}/.github/workflows/post_coverage_to_pr.py + rm -f .coverage coverage.json + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2.6.2 + if: github.event_name == 'pull_request' + with: + recreate: true + path: /tmp/${{ matrix.location.name }}/index.html + + - name: Write code coverage results to Job Summary + run: cat /tmp/${{ matrix.location.name }}-code-coverage-results.md >> $GITHUB_STEP_SUMMARY + + - name: Generate requirements.txt from setup.py for use in security checks + continue-on-error: true + shell: bash + run: | + pipx run pipreqs --ignore bin,etc,include,lib,lib64 --pypi-server ${{ env.PIP_INDEX_URL }} --print ${{ matrix.location.directory }} 2>/dev/null > "/tmp/requirements-${{ matrix.location.name }}.txt" + + - name: Check for security issues + uses: pypa/gh-action-pip-audit@v1.0.7 + continue-on-error: true + with: + extra-index-urls: ${{ env.PIP_INDEX_URL }} + inputs: "/tmp/requirements-${{ matrix.location.name }}.txt" + local: true + + - name: Lint for Python syntax errors or undefined names + if: github.event_name == 'pull_request' + continue-on-error: true + shell: bash + run: ruff --quiet check --exit-zero --format text ${{ matrix.location.directory }} | lintly --exit-zero --fail-on new --no-post-status --request-changes --use-checks --api-key ${{ secrets.GITHUB_TOKEN }} --pr ${{ github.event.number }} --repo ${{ github.repository }} --commit-sha=${{ github.event.pull_request.head.sha }} --format unix + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and deploy to Dagster Cloud hybrid + uses: dagster-io/dagster-cloud-action/actions/hybrid_branch_deploy@v0.1 + with: + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + checkout_repo: false + location: ${{ toJson(matrix.location) }} + secrets: | + pipconf=/home/runner/.config/pip/pip.conf + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_BUILDKIT: 1 + + # - name: Additional image tags for deployment environment and Klar Central Station + # shell: bash + # continue-on-error: true + # working-directory: ${{ matrix.location.directory }} + # run: | + # docker tag "${{ matrix.location.registry }}:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}" "${{ matrix.location.registry }}:${{ github.run_number }}-${{ github.sha }}" + # docker push "${{ matrix.location.registry }}:${{ github.run_number }}-${{ github.sha }}" + + infra: + name: Push changes in the infra/environments-stack into Kubernetes + runs-on: [self-hosted, default] + needs: setup + concurrency: + group: "terraform-${{ matrix.environment }}" + strategy: + fail-fast: false + matrix: + environment: [staging] + + steps: + - uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: '${{ secrets.ORG_SSH_KEY }}' + - uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/checkout@v3 + with: + repository: 'klar-mx/github-actions' + ref: 'v2' + path: '.actions' + ssh-key: '${{ secrets.ORG_SSH_KEY }}' + - uses: ./.actions/deploy_infra + env: + TF_VAR_git_branch_name: ${{ env.GIT_BRANCH_NAME }} + with: + environment: '${{ matrix.environment }}' + image: '739282534308.dkr.ecr.us-east-2.amazonaws.com/{{ github.event.repository.name }}:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}' diff --git a/main.yml b/main.yml new file mode 100644 index 0000000..4c50534 --- /dev/null +++ b/main.yml @@ -0,0 +1,52 @@ +name: "Hello" +on: + push: + branches: [ main, master ] + pull_request: + types: [opened, reopened, synchronize] + +permissions: + contents: read + issues: write + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +env: + GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SHA: ${{ github.sha }} + +jobs: + build_job: + runs-on: ubuntu-latest + name: Build on ${{ matrix.distro }} (${{ matrix.version }}) for ${{ matrix.arch }} + + strategy: + matrix: + include: + - arch: x86_64 + distro: Debian + version: bullseye + # - arch: aarch64 + # distro: Debian + # version: bullseye + # - arch: ppc64le + # distro: Debian + # version: bullseye + # - arch: risc64 + # distro: Ubuntu + # version: ubuntu_latest + + id: build + steps: + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 + with: + CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Print nixpkgs version + run: nix-instantiate --eval -E '(import {}).lib.version' + - run: nix build + - run: nix flake check