greg/hello
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

WIP

Browse source
This commit is contained in:
Greg Burd 2023-09-19 16:37:58 -04:00
parent bb933c22b7
commit d2000eed3a
No known key found for this signature in database
GPG key ID: 1FC1E7793410DE46
7 changed files with 552 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
.github/TODO vendored Normal file
View file

@ -0,0 +1,65 @@
steps:
- uses: actions/checkout@v3
- uses: uraimo/run-on-arch-action@v2
if: ${{ matrix.arch }} != x86_64
name: Build artifact
id: build
with:
arch: ${{ matrix.arch }}
distro: ${{ matrix.distro }}
# Not required, but speeds up builds
githubToken: ${{ github.token }}
# Set an output parameter `uname` for use in subsequent steps
run: |
uname -a
echo ::set-output name=uname::$(uname -a)
# Create an artifacts directory
setup: |
mkdir -p "${PWD}/artifacts"
# Mount the artifacts directory as /artifacts in the container
dockerRunArgs: |
--volume "${PWD}/artifacts:/artifacts"
# Pass some environment variables to the container
env: | # YAML, but pipe character is necessary
artifact_name: git-${{ matrix.distro }}_${{ matrix.arch }}
# The shell to run commands with in the container
shell: /bin/sh
# Install some dependencies in the container. This speeds up builds if
# you are also using githubToken. Any dependencies installed here will
# be part of the container image that gets cached, so subsequent
# builds don't have to re-install them. The image layer is cached
# publicly in your project's package repository, so it is vital that
# no secrets are present in the container state or logs.
install: |
case "${{ matrix.distro }}" in
ubuntu*|jessie|stretch|buster|bullseye)
apt-get update -q -y
apt-get install -q -y git
;;
fedora*)
dnf -y update
dnf -y install git which
;;
alpine*)
apk update
apk add git
;;
esac
# Produce a binary artifact and place it in the mounted volume
run: |
cp $(which git) "/artifacts/${artifact_name}"
echo "Produced artifact at /artifacts/${artifact_name}"
- name: Show the artifact
# Items placed in /artifacts in the container will be in
# ${PWD}/artifacts on the host.
run: |
ls -al "${PWD}/artifacts"

21
.github/workflows/main.yml vendored
View file

@ -9,9 +9,9 @@ permissions:
contents: read
issues: write
# concurrency:
# group: ${{ github.ref }}
# cancel-in-progress: true
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
env:
GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
@ -19,14 +19,27 @@ env:
GITHUB_SHA: ${{ github.sha }}
jobs:
nix-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v22
- id: set-matrix
name: Generate Nix Matrix
run: |
set -Eeu
echo "matrix=$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT"
nix-build:
needs: nix-matrix
name: Build on ${{ matrix.attr }}
runs-on: ${{ matrix.os }}
strategy:
matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}}
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v22
- uses: actions/checkout@v3
- run: nix build -L ".#${{ matrix.attr }}"
# runs-on: ubuntu-latest

21
.github/workflows/nix-github-actions.yml vendored
View file

@ -1,21 +0,0 @@
name: Nix Flake actions
on:
pull_request:
types: [opened, reopened, synchronize]
push:
branches: [main, master]
jobs:
nix-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v22
- id: set-matrix
name: Generate Nix Matrix
run: |
set -Eeu
echo "matrix=$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT"

73
NOTES Normal file
View file

@ -0,0 +1,73 @@
platforms = [
"i686-cygwin"
"x86_64-cygwin"
"x86_64-darwin"
"i686-darwin"
"aarch64-darwin"
"armv7a-darwin"
"i686-freebsd13"
"x86_64-freebsd13"
"aarch64-genode"
"i686-genode"
"x86_64-genode"
"x86_64-solaris"
"js-ghcjs"
"aarch64-linux"
"armv5tel-linux"
"armv6l-linux"
"armv7a-linux"
"armv7l-linux"
"i686-linux"
"m68k-linux"
"microblaze-linux"
"microblazeel-linux"
"mipsel-linux"
"mips64el-linux"
"powerpc64-linux"
"powerpc64le-linux"
"riscv32-linux"
"riscv64-linux"
"s390-linux"
"s390x-linux"
"x86_64-linux"
"mmix-mmixware"
"aarch64-netbsd"
"armv6l-netbsd"
"armv7a-netbsd"
"armv7l-netbsd"
"i686-netbsd"
"m68k-netbsd"
"mipsel-netbsd"
"powerpc-netbsd"
"riscv32-netbsd"
"riscv64-netbsd"
"x86_64-netbsd"
"aarch64_be-none"
"aarch64-none"
"arm-none"
"armv6l-none"
"avr-none"
"i686-none"
"microblaze-none"
"microblazeel-none"
"msp430-none"
"or1k-none"
"m68k-none"
"powerpc-none"
"powerpcle-none"
"riscv32-none"
"riscv64-none"
"rx-none"
"s390-none"
"s390x-none"
"vc4-none"
"x86_64-none"
"i686-openbsd"
"x86_64-openbsd"
"x86_64-redox"
"wasm64-wasi"
"wasm32-wasi"
"x86_64-windows"
"i686-windows"
];

51
_flake.nix Normal file
View file

@ -0,0 +1,51 @@
# Tests run by 'nix flake check' and by Hydra.
checks = forAllSystems
(system:
with pkgs.${system};
{
inherit (self.packages.${system}) hello;
# Additional tests, if applicable.
test = stdenv.mkDerivation {
pname = "hello-test";
inherit version;
buildInputs = [ hello ];
dontUnpack = true;
buildPhase = ''
echo 'running some integration tests'
[[ $(hello) = 'Hello Nixers!' ]]
'';
installPhase = "mkdir -p $out";
};
}
// lib.optionalAttrs stdenv.isLinux {
# A VM test of the NixOS module.
vmTest =
with import (nixpkgs + "/nixos/lib/testing-python.nix") {
inherit system;
};
makeTest {
nodes = {
client = { ... }: {
imports = [ self.nixosModules.hello ];
};
};
testScript =
''
start_all()
client.wait_for_unit("multi-user.target")
client.succeed("hello")
'';
};
}
);
};

294
branch_deployments.yml_ Normal file
View file

@ -0,0 +1,294 @@
name: Deploy a development branch PR using Dagster Hybrid Cloud
# on:
# pull_request:
# types: [opened, synchronize, reopened]
concurrency:
# Cancel in-progress runs on same branch
group: ${{ github.ref }}
cancel-in-progress: true
env:
AWS_PROFILE: "klar-neutral"
AWS_ACCOUNT_ID: "739282534308"
AWS_REGION: "us-east-2"
DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
DAGSTER_CLOUD_ORGANIZATION_ID: "klar"
DAGSTER_CLOUD_URL: "https://klar.dagster.cloud"
DOCKER_CONFIG: $GITHUB_WORKSPACE
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_SHA: ${{ github.sha }}
PIP_ROOT_USER_ACTION: ignore
GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} # For PRs, use the head branch name, for other commits, use the branch name itself
jobs:
setup:
name: Parse Dagster Hybrid Cloud configuration file
runs-on: [self-hosted, default]
outputs:
locations: ${{ steps.load_workspace_file.outputs.build_info }}
secrets_set: ${{ steps.load_workspace_file.outputs.secrets_set }}
steps:
- name: Checkout
uses: actions/checkout@v3
- run: |
git fetch origin ${{ github.event.repository.default_branch }}:${{ github.event.repository.default_branch }}
git fetch origin ${{ env.GIT_BRANCH_NAME }}:${{ env.GIT_BRANCH_NAME }}
- name: Configure PIP to use AWS/CodeArtifact
run: |
CODEARTIFACT_AUTH_TOKEN=$(aws --profile klar-neutral codeartifact get-authorization-token --domain klar-neutral --query authorizationToken --output text)
pip config set global.index-url "https://aws:${CODEARTIFACT_AUTH_TOKEN}@klar-neutral-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-2.amazonaws.com/pypi/klar/simple/"
- name: Install Virtual Environment for Python
shell: bash
run: python3 -m pip install virtualenv
- name: Setup Virtual Environment for Python
shell: bash
run: virtualenv /tmp/venv
- name: Activate Virtual Environment for Python
shell: bash
run: |
. /tmp/venv/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
- name: Display Python version
shell: bash
run: |
which python
python -c "import sys; print(sys.version)"
- name: Upgrade pip
shell: bash
run: |
python -m pip install --upgrade pip
- name: Install required Python dependencies
shell: bash
run: |
pip install --upgrade oyaml sh GitPython
- name: Create build_info with only the changed code locations
id: load_workspace_file
shell: bash
env:
CHANGES_ONLY: yes
DEFAULT_BRANCH_NAME: ${{ github.event.repository.default_branch }}
run: |
python ${GITHUB_WORKSPACE}/.github/workflows/parse_workspace.py dagster_cloud.yaml ${{ env.GIT_BRANCH_NAME }} >> $GITHUB_OUTPUT
build:
name: Dagster Hybrid Deploy
runs-on: [self-hosted, default]
needs: setup
if: ${{ needs.setup.outputs.locations != '[]' }}
permissions: write-all
strategy:
fail-fast: false
matrix:
location: ${{ fromJSON(needs.setup.outputs.locations) }}
python-version: ["3.10"]
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.sha }}
lfs: true
# submodules: recursive
- name: Copy common utils framework into this code location
shell: bash
working-directory: ${{ matrix.location.directory }}
run: |
[ ! -d utils ] && mkdir utils
(cd ${GITHUB_WORKSPACE} && tar -cf - utils) | tar -xvf -
- name: Set deps-cache-from
# Don't use cached deps if instructed
shell: bash
if: ${{ inputs.force_rebuild_deps != 'true' }}
# For PR commits, use the target branch name as the cache-tag, for other commits, use the branch name itself
run: echo "FLAG_DEPS_CACHE_FROM=--deps-cache-from=${{ github.repository }}/${{ matrix.location.name }}/${{ github.base_ref && github.base_ref || github.ref_name }}" >> $GITHUB_ENV
- name: Set deps-cache-to
# Only write to the cache-tag for non PR commits so PR commits don't upgrade dependency versions
shell: bash
if: ${{ github.base_ref == '' }}
run: echo "FLAG_DEPS_CACHE_TO=--deps-cache-to=${{ github.repository }}/${{ matrix.location.name }}/${{ github.ref_name }}" >> $GITHUB_ENV
- name: Configure PIP to use AWS/CodeArtifact
run: |
export CODEARTIFACT_AUTH_TOKEN=$(aws --profile klar-neutral codeartifact get-authorization-token --domain klar-neutral --query authorizationToken --output text)
echo "PIP_INDEX_URL=https://aws:${CODEARTIFACT_AUTH_TOKEN}@klar-neutral-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-2.amazonaws.com/pypi/klar/simple/" >> $GITHUB_ENV
pip config set global.index-url "https://aws:${CODEARTIFACT_AUTH_TOKEN}@klar-neutral-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-2.amazonaws.com/pypi/klar/simple/"
- name: Login to AWS/ECR
shell: bash
run: |
aws ecr get-login-password --region ${{ env.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com
- name: Create ECR repository if it doesn't exist
id: create-repository
continue-on-error: true
shell: bash
run: |
output=$(aws ecr describe-repositories --repository-names "${{ matrix.location.name }}" 2>&1)
if [ $? -ne 0 ]; then
if echo "${output}" | grep -q RepositoryNotFoundException; then
aws ecr create-repository --repository-name "${{ matrix.location.name }}"
else
>&2 echo "${output}"
fi
fi
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Install Wheel, SetupTools, Virtual Environment, and PIPx
shell: bash
run: |
python3 -m pip install wheel setuptools virtualenv pipx
- name: Activate Virtual Environment for code location
shell: bash
working-directory: ${{ matrix.location.directory }}
run: |
virtualenv venv-${{ matrix.location.name }}
source ./venv-${{ matrix.location.name }}/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
- name: Display Python version
shell: bash
run: |
which python
python -c "import sys; print(sys.version)"
- name: Upgrade pip
shell: bash
run: |
python -m pip install --upgrade pip
- name: Install more Python dependencies
shell: bash
run: |
pip install coverage pytest pytest-json-report pytest-md-report pytest-annotate pyannotate ruff markupsafe==2.0.1 lintly
pip install oyaml sh GitPython
- name: Install code location requirements
working-directory: ${{ matrix.location.directory }}
shell: bash
run: |
if [ -d utils ]; then
cp requirements.txt /tmp
sed -i -e "s|/opt/dagster/app|$(pwd)|" requirements.txt
pip install -e utils
mv /tmp/requirements.txt .
fi
pip install -e ".[tests]"
- name: Run tests and collect coverage
working-directory: ${{ matrix.location.directory }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
python -m coverage run -m pytest *.py --doctest-modules --ignore=${{ matrix.location.name }} \
--md-report --md-report-output="/tmp/${{ matrix.location.name }}-code-coverage-results.md"
python -m coverage html --data-file=.coverage --directory=/tmp/${{ matrix.location.name }} --title=${{ matrix.location.name }} --rcfile=${GITHUB_WORKSPACE}/.coveragerc
python -m coverage json --data-file=.coverage --o coverage.json --rcfile=${GITHUB_WORKSPACE}/.coveragerc
python ${GITHUB_WORKSPACE}/.github/workflows/post_coverage_to_pr.py
rm -f .coverage coverage.json
- name: Add Coverage PR Comment
uses: marocchino/sticky-pull-request-comment@v2.6.2
if: github.event_name == 'pull_request'
with:
recreate: true
path: /tmp/${{ matrix.location.name }}/index.html
- name: Write code coverage results to Job Summary
run: cat /tmp/${{ matrix.location.name }}-code-coverage-results.md >> $GITHUB_STEP_SUMMARY
- name: Generate requirements.txt from setup.py for use in security checks
continue-on-error: true
shell: bash
run: |
pipx run pipreqs --ignore bin,etc,include,lib,lib64 --pypi-server ${{ env.PIP_INDEX_URL }} --print ${{ matrix.location.directory }} 2>/dev/null > "/tmp/requirements-${{ matrix.location.name }}.txt"
- name: Check for security issues
uses: pypa/gh-action-pip-audit@v1.0.7
continue-on-error: true
with:
extra-index-urls: ${{ env.PIP_INDEX_URL }}
inputs: "/tmp/requirements-${{ matrix.location.name }}.txt"
local: true
- name: Lint for Python syntax errors or undefined names
if: github.event_name == 'pull_request'
continue-on-error: true
shell: bash
run: ruff --quiet check --exit-zero --format text ${{ matrix.location.directory }} | lintly --exit-zero --fail-on new --no-post-status --request-changes --use-checks --api-key ${{ secrets.GITHUB_TOKEN }} --pr ${{ github.event.number }} --repo ${{ github.repository }} --commit-sha=${{ github.event.pull_request.head.sha }} --format unix
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build and deploy to Dagster Cloud hybrid
uses: dagster-io/dagster-cloud-action/actions/hybrid_branch_deploy@v0.1
with:
dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
checkout_repo: false
location: ${{ toJson(matrix.location) }}
secrets: |
pipconf=/home/runner/.config/pip/pip.conf
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKER_BUILDKIT: 1
# - name: Additional image tags for deployment environment and Klar Central Station
# shell: bash
# continue-on-error: true
# working-directory: ${{ matrix.location.directory }}
# run: |
# docker tag "${{ matrix.location.registry }}:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}" "${{ matrix.location.registry }}:${{ github.run_number }}-${{ github.sha }}"
# docker push "${{ matrix.location.registry }}:${{ github.run_number }}-${{ github.sha }}"
infra:
name: Push changes in the infra/environments-stack into Kubernetes
runs-on: [self-hosted, default]
needs: setup
concurrency:
group: "terraform-${{ matrix.environment }}"
strategy:
fail-fast: false
matrix:
environment: [staging]
steps:
- uses: webfactory/ssh-agent@v0.8.0
with:
ssh-private-key: '${{ secrets.ORG_SSH_KEY }}'
- uses: actions/checkout@v3
with:
ref: ${{ github.sha }}
fetch-depth: 0
- uses: actions/checkout@v3
with:
repository: 'klar-mx/github-actions'
ref: 'v2'
path: '.actions'
ssh-key: '${{ secrets.ORG_SSH_KEY }}'
- uses: ./.actions/deploy_infra
env:
TF_VAR_git_branch_name: ${{ env.GIT_BRANCH_NAME }}
with:
environment: '${{ matrix.environment }}'
image: '739282534308.dkr.ecr.us-east-2.amazonaws.com/{{ github.event.repository.name }}:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}'

52
main.yml Normal file
View file

@ -0,0 +1,52 @@
name: "Hello"
on:
push:
branches: [ main, master ]
pull_request:
types: [opened, reopened, synchronize]
permissions:
contents: read
issues: write
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
env:
GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_SHA: ${{ github.sha }}
jobs:
build_job:
runs-on: ubuntu-latest
name: Build on ${{ matrix.distro }} (${{ matrix.version }}) for ${{ matrix.arch }}
strategy:
matrix:
include:
- arch: x86_64
distro: Debian
version: bullseye
# - arch: aarch64
# distro: Debian
# version: bullseye
# - arch: ppc64le
# distro: Debian
# version: bullseye
# - arch: risc64
# distro: Ubuntu
# version: ubuntu_latest
id: build
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v22
with:
CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: Print nixpkgs version
run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
- run: nix build
- run: nix flake check