steps:

- kind: buildImage2

name: git_make_bash_image

publishAs: git-make-bash

resources:

storage: 10Gi

cpu: "2"

memory: standard

dockerFile:

inline: |

ARG BASE_IMAGE={{ global.docker_root_image }}

FROM $BASE_IMAGE

RUN apt-get update && apt-get install -y git make bash

- kind: runImage

name: merge_code

image:

valueFrom: git_make_bash_image.image

script: |

set -ex

cd /io

mkdir -p repo

cd repo

{{ code.checkout_script }}

make -C hail python-version-info HAIL_RELEASE_MODE=1 MILLOPTS=--no-server

git rev-parse HEAD > git_version

outputs:

- from: /io/repo/

to: /repo

- from: /io/repo/hail/python/hail/hail_version

to: /hail_version

- from: /io/repo/hail/python/hail/hail_pip_version

to: /hail_pip_version

- from: /io/repo/git_version

to: /git_version

dependsOn:

- git_make_bash_image

- kind: createNamespace

name: default_ns

namespaceName: default

secrets:

- name: auth-oauth2-client-secret

- name: hail-ci-0-1-github-oauth-token

- name: testns-test-gsa-key

- name: testns-test-dev-gsa-key

- name: testns-auth-gsa-key

- name: testns-batch-gsa-key

- name: testns-ci-gsa-key

- name: testns-grafana-gsa-key

- name: test-aws-key

clouds:

- gcp

- name: test-azure-key

clouds:

- gcp

- name: zulip-config

- name: billing-monitor-gsa-key

clouds:

- gcp

- name: hail-ci-0-1-service-account-key

- name: test-dataproc-service-account-key

clouds:

- gcp

- name: batch-worker-ssh-public-key

clouds:

- azure

- kind: runImage

name: copy_third_party_images

image: quay.io/skopeo/stable:v1.13.2

script: |

set -ex

retry() {

"$@" ||

(sleep 2 && "$@") ||

(sleep 5 && "$@");

}

REGISTRY={{ global.docker_prefix.split('/')[0] }}

{% if global.cloud == "gcp" %}

cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY

{% elif global.cloud == "azure" %}

dnf install -y jq

USERNAME=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.appId')

cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY

{% else %}

echo "unknown cloud {{ global.cloud }}"

exit 1

{% endif %}

cd /io/docker/third-party

DOCKER_PREFIX={{ global.docker_prefix }} retry bash copy_images.sh

inputs:

- from: /repo/docker

to: /io/docker

scopes:

- deploy

dependsOn:

- default_ns

- merge_code

- kind: buildImage2

name: hail_ubuntu_image

dockerFile: /io/hail-ubuntu/Dockerfile

contextPath: /io/hail-ubuntu

publishAs: hail-ubuntu

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/docker/hail-ubuntu

to: /io/hail-ubuntu

dependsOn:

- merge_code

- copy_third_party_images

- kind: buildImage2

name: hail_ubuntu_image_python_3_10

dockerFile: /io/hail-ubuntu/Dockerfile

contextPath: /io/hail-ubuntu

publishAs: hail-ubuntu-python-3-10

buildArgs:

- name: PYTHON_VERSION

value: "3.10"

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/docker/hail-ubuntu

to: /io/hail-ubuntu

dependsOn:

- merge_code

- copy_third_party_images

- kind: buildImage2

name: hail_ubuntu_image_python_3_11

dockerFile: /io/hail-ubuntu/Dockerfile

contextPath: /io/hail-ubuntu

publishAs: hail-ubuntu-python-3-11

buildArgs:

- name: PYTHON_VERSION

value: "3.11"

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/docker/hail-ubuntu

to: /io/hail-ubuntu

dependsOn:

- merge_code

- copy_third_party_images

- kind: deploy

name: deploy_batch_sa

namespace:

valueFrom: default_ns.name

config: batch/service-account.yaml

dependsOn:

- default_ns

- kind: deploy

name: deploy_ci_agent

namespace:

valueFrom: default_ns.name

config: ci/ci-agent.yaml

dependsOn:

- default_ns

- kind: deploy

name: deploy_test_batch_sa

namespace:

valueFrom: default_ns.name

config: batch/test-sa.yaml

dependsOn:

- default_ns

- kind: buildImage2

name: base_image

dockerFile: /io/repo/docker/Dockerfile.base

contextPath: /io/repo

publishAs: base

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/docker/Dockerfile.base

to: /io/repo/docker/Dockerfile.base

dependsOn:

- hail_ubuntu_image

- merge_code

- kind: buildImage2

name: create_certs_image

dockerFile: /io/tls/Dockerfile

contextPath: /io/tls

publishAs: create_certs_image

inputs:

- from: /repo/tls

to: /io/tls

dependsOn:

- hail_ubuntu_image

- merge_code

- kind: runImage

name: create_ssl_config_hail_root

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: create_certs_image.image

script: |

{% if not deploy %}

kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root \

--template={% raw %}'{{index .data "hail-root-cert.pem"}}'{% endraw %} \

| base64 --decode \

| openssl x509 -checkend 0 -noout -in -

if [ "$?" -ne 0 ]

then

kubectl delete secret -n {{ default_ns.name }} ssl-config-hail-root

fi

{% endif %}

openssl req -new -x509 -subj /CN=hail-root -nodes -newkey rsa:4096 -keyout hail-root-key.pem -out hail-root-cert.pem

until kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root

do

kubectl create secret generic -n {{ default_ns.name }} ssl-config-hail-root \

--from-file=hail-root-key.pem \

--from-file=hail-root-cert.pem

done

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

scopes:

- test

- dev

dependsOn:

- default_ns

- create_certs_image

- kind: runImage

name: create_certs

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: create_certs_image.image

script: |

set -ex

python3 create_certs.py \

{{ default_ns.name }} \

config.yaml \

/ssl-config-hail-root/hail-root-key.pem \

/ssl-config-hail-root/hail-root-cert.pem

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

secrets:

- name: ssl-config-hail-root

namespace:

valueFrom: default_ns.name

mountPath: /ssl-config-hail-root

dependsOn:

- default_ns

- create_certs_image

- create_ssl_config_hail_root

- kind: buildImage2

name: ci_utils_image

dockerFile: /io/repo/ci/Dockerfile.ci-utils

contextPath: /io/repo

publishAs: ci-utils

resources:

storage: 20Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/ci

to: /io/repo/ci

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail/python/hailtop/hail_version

- from: /repo/gear

to: /io/repo/gear

dependsOn:

- base_image

- merge_code

- kind: runImage

name: create_test_database_server_config

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: create_certs_image.image

script: |

set -ex

if ! kubectl get secret -n {{ default_ns.name }} database-server-config;

then

NAMESPACE={{ default_ns.name }} bash /create_test_db_config.sh

fi

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

scopes:

- dev

- test

dependsOn:

- default_ns

- create_certs_image

- kind: deploy

name: deploy_test_db

namespace:

valueFrom: default_ns.name

config: docker/mysql/db.yaml

wait:

- kind: Service

name: db

for: alive

resource_type: statefulset

scopes:

- dev

- test

dependsOn:

- default_ns

- create_test_database_server_config

- kind: buildImage2

name: admin_pod_image

dockerFile: /io/repo/admin-pod/Dockerfile

contextPath: /io/repo

publishAs: admin-pod

inputs:

- from: /repo/admin-pod

to: /io/repo/admin-pod

dependsOn:

- hail_ubuntu_image

- kind: buildImage2

name: auth_image

dockerFile: /io/repo/auth/Dockerfile

contextPath: /io/repo

publishAs: auth

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/auth

to: /io/repo/auth

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail/python/hailtop/hail_version

- from: /repo/gear

to: /io/repo/gear

- from: /repo/web_common

to: /io/repo/web_common

dependsOn:

- hail_ubuntu_image

- merge_code

- kind: runImage

name: delete_auth_tables

image:

valueFrom: auth_image.image

script: |

set -ex

cd /io

python3 delete_dev_database.py {{ default_ns.name }} auth

inputs:

- from: /repo/ci/delete_dev_database.py

to: /io/delete_dev_database.py

secrets:

- name: database-server-config

namespace:

valueFrom: default_ns.name

mountPath: /sql-config

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

network: private

runIfRequested: true

scopes:

- dev

dependsOn:

- default_ns

- auth_image

- merge_code

- create_test_database_server_config

- kind: createDatabase2

name: auth_database

databaseName: auth

image:

valueFrom: ci_utils_image.image

migrations:

- name: initial

script: /io/sql/initial.sql

- name: copy-paste-tokens

script: /io/sql/copy-paste-tokens.sql

- name: drop-bucket

script: /io/sql/drop-bucket.sql

- name: add-trial-billing-project

script: /io/sql/add-trial-billing-project.sql

- name: add-roles

script: /io/sql/add-roles.sql

- name: rename-gsa-identity

script: /io/sql/rename-gsa-identity.sql

- name: support-azure-oauth

script: /io/sql/support-azure-oauth.sql

- name: change-username-collation

script: /io/sql/change-username-collation.sql

- name: add-hail-identity-uid

script: /io/sql/add-hail-identity-uid.sql

inputs:

- from: /repo/auth/sql

to: /io/sql

namespace:

valueFrom: default_ns.name

shutdowns:

- kind: Deployment

namespace:

valueFrom: default_ns.name

name: auth

dependsOn:

- default_ns

- merge_code

- delete_auth_tables

- ci_utils_image

- create_test_database_server_config

- deploy_test_db

- kind: runImage

name: create_deploy_config

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: ci_utils_image.image

script: |

set -ex

# k8s deploy config

{% if default_ns.name == "default" %}

cat > deploy-config.json <<EOF

{"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"{{ global.domain }}"}

EOF

{% else %}

cat > deploy-config.json <<EOF

{"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"internal.{{ global.domain }}","base_path":"/{{ default_ns.name }}"}

EOF

{% endif %}

kubectl -n {{ default_ns.name }} create secret generic deploy-config \

--from-file=./deploy-config.json \

--save-config --dry-run=client -o yaml \

| kubectl -n {{ default_ns.name }} apply -f -

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

dependsOn:

- default_ns

- ci_utils_image

- kind: deploy

name: deploy_admin_pod

namespace:

valueFrom: default_ns.name

config: admin-pod/admin-pod.yaml

scopes:

- deploy

- dev

dependsOn:

- default_ns

- admin_pod_image

- create_deploy_config

- create_certs

- kind: runImage

name: create_session_key

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: ci_utils_image.image

script: |

set -ex

# create session keys

N=$(kubectl -n {{ default_ns.name }} get secret --ignore-not-found=true --no-headers session-secret-key | wc -l | tr -d '[:space:]')

if [[ $N != 0 ]]; then

exit

fi

mkdir /session-secret-key

cat > generate-session-key.py <<EOF

import base64

from cryptography import fernet

with open('/session-secret-key/session-secret-key', 'wb') as f:

f.write(base64.urlsafe_b64decode(fernet.Fernet.generate_key()))

EOF

python3 generate-session-key.py

kubectl -n {{ default_ns.name }} create secret generic session-secret-key --from-file=/session-secret-key/session-secret-key

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

dependsOn:

- default_ns

- ci_utils_image

- kind: runImage

name: create_test_gsa_keys

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: ci_utils_image.image

script: |

kubectl -n {{ default_ns.name }} get -o json secret testns-test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -

kubectl -n {{ default_ns.name }} get -o json secret testns-test-dev-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-dev-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -

kubectl -n {{ default_ns.name }} get -o json secret testns-auth-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "auth-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -

kubectl -n {{ default_ns.name }} get -o json secret testns-batch-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "batch-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -

kubectl -n {{ default_ns.name }} get -o json secret testns-ci-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "ci-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -

kubectl -n {{ default_ns.name }} get -o json secret testns-grafana-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "grafana-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -

scopes:

- test

- dev

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

dependsOn:

- default_ns

- ci_utils_image

- kind: deploy

name: deploy_auth_driver_service_account

namespace:

valueFrom: default_ns.name

config: auth/auth-driver-service-account.yaml

dependsOn:

- default_ns

- kind: runImage

name: create_accounts

resources:

memory: standard

cpu: '0.25'

image:

valueFrom: auth_image.image

script: |

set -ex

export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

export HAIL_SCOPE={{ scope }}

export GOOGLE_APPLICATION_CREDENTIALS=/auth-gsa-key/key.json

python3 /io/bootstrap_create_accounts.py

serviceAccount:

name: admin

namespace:

valueFrom: default_ns.name

network: private

secrets:

- name:

valueFrom: auth_database.user_secret_name

namespace:

valueFrom: default_ns.name

mountPath: /sql-config

- name: database-server-config

namespace:

valueFrom: default_ns.name

mountPath: /database-server-config

- name: auth-gsa-key

namespace:

valueFrom: default_ns.name

mountPath: /auth-gsa-key

- name: global-config

namespace:

valueFrom: default_ns.name

mountPath: /global-config

inputs:

- from: /repo/ci/bootstrap_create_accounts.py

to: /io/bootstrap_create_accounts.py

dependsOn:

- default_ns

- deploy_test_batch_sa

- auth_database

- auth_image

- create_deploy_config

- deploy_auth_driver_service_account

- create_test_gsa_keys

- create_test_database_server_config

- kind: buildImage2

name: hailgenetics_vep_grch37_85_image

dockerFile: /io/repo/docker/hailgenetics/vep/grch37/85/Dockerfile

contextPath: /io/repo/docker/vep/

publishAs: hailgenetics/vep-grch37-85

inputs:

- from: /repo

to: /io/repo

dependsOn:

- merge_code

- hail_ubuntu_image

- kind: buildImage2

name: hailgenetics_vep_grch38_95_image

dockerFile: /io/repo/docker/hailgenetics/vep/grch38/95/Dockerfile

contextPath: /io/repo/docker/vep/

publishAs: hailgenetics/vep-grch38-95

inputs:

- from: /repo

to: /io/repo

dependsOn:

- merge_code

- hail_ubuntu_image

- kind: buildImage2

name: monitoring_image

dockerFile: /io/repo/monitoring/Dockerfile

contextPath: /io/repo

publishAs: monitoring

inputs:

- from: /repo/monitoring

to: /io/repo/monitoring

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail/python/hailtop/hail_version

- from: /repo/gear

to: /io/repo/gear

- from: /repo/web_common

to: /io/repo/web_common

dependsOn:

- hail_ubuntu_image

- merge_code

- kind: buildImage2

name: batch_image

dockerFile: /io/repo/batch/Dockerfile

contextPath: /io/repo

publishAs: batch

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/batch

to: /io/repo/batch

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail/python/hailtop/hail_version

- from: /repo/gear

to: /io/repo/gear

- from: /repo/web_common

to: /io/repo/web_common

dependsOn:

- hail_ubuntu_image

- merge_code

- kind: buildImage2

name: hailgenetics_hailtop_image

dockerFile: /io/repo/docker/hailgenetics/hailtop/Dockerfile

contextPath: /io/repo

publishAs: hailgenetics/hailtop

inputs:

- from: /repo/docker/hailgenetics/hailtop

to: /io/repo/docker/hailgenetics/hailtop

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail_version

dependsOn:

- merge_code

- hail_ubuntu_image

- kind: buildImage2

name: ci_image

dockerFile: /io/repo/ci/Dockerfile

contextPath: /io/repo

publishAs: ci

inputs:

- from: /repo/ci

to: /io/repo/ci

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail/python/hailtop/hail_version

- from: /repo/gear

to: /io/repo/gear

- from: /repo/web_common

to: /io/repo/web_common

dependsOn:

- hail_ubuntu_image

- merge_code

- kind: buildImage2

name: hail_buildkit_image

dockerFile: /io/repo/ci/buildkit/Dockerfile

contextPath: /io/repo/ci

publishAs: hail-buildkit

inputs:

- from: /repo/ci

to: /io/repo/ci

dependsOn:

- merge_code

- copy_third_party_images

- kind: runImage

name: build_hail_jar_and_wheel

image:

valueFrom: base_image.image

resources:

memory: standard

cpu: '2'

script: |

set -ex

cd /io/repo/hail

export MILLOPTS='--no-server' HAIL_RELEASE_MODE=1

time retry sh millw --no-server --version

# We've encountered the following sporadic error in CI between `mill`

# finishing assembling the jar and `Make` copying the jar:

#

# cp -f out/assembly.dest/out.jar python/hail/backend/hail-all-spark.jar

# cp: cannot stat 'out/assembly.dest/out.jar': No such file or directory

#

# This was likely from running mill in server mode; mill main exits before

# the jar has finished being written.

time retry make shadowJar

if [ ! -f out/assembly.dest/out.jar ]; then

echo 'no out.jar found after mill assembly returned. going to sleep'

sleep 5

fi

time retry make wheel

# Check wheel size is small enough for pypi (< 200 MiB)

HAIL_PIP_VERSION=$(cat python/hail/hail_pip_version)

WHEEL_PATH="build/deploy/dist/hail-$HAIL_PIP_VERSION-py3-none-any.whl"

du -h $WHEEL_PATH

$(python3 -c "import os; exit(1) if (os.path.getsize('$WHEEL_PATH')) > (200 * 1024 * 1024) else exit(0)")

inputs:

- from: /repo

to: /io/repo

outputs:

- from: /io/repo/hail/out

to: /derived/release/hail/out/

- from: /io/repo/hail/build/deploy/dist

to: /derived/release/hail/build/deploy/dist

dependsOn:

- base_image

- merge_code

- kind: runImage

name: build_hail_debug_jar_and_wheel

image:

valueFrom: base_image.image

resources:

memory: standard

cpu: '2'

script: |

set -ex

cd /io/repo/hail

export MILLOPTS='--no-server'

time retry sh millw --no-server --version

# See `build_hail_jar_and_wheel`

time retry make shadowJar

if [ ! -f out/assembly.dest/out.jar ]; then

echo 'no out.jar found after mill assembly returned. going to sleep'

sleep 5

fi

time retry make wheel

inputs:

- from: /repo

to: /io/repo

outputs:

- from: /io/repo/hail/out

to: /derived/debug/hail/out

- from: /io/repo/hail/build/deploy/dist

to: /derived/debug/hail/build/deploy/dist

dependsOn:

- base_image

- merge_code

- kind: runImage

name: build_debug_hail_test_jar

image:

valueFrom: base_image.image

resources:

memory: standard

cpu: '2'

script: |

set -ex

cd /io/repo/hail

export MILLOPTS='--no-server'

time retry sh millw --no-server --version

# See `build_hail_jar_and_wheel`

time retry make shadowTestJar

if [ ! -f out/test/assembly.dest/out.jar ]; then

echo 'no out.jar found after mill assembly returned. going to sleep'

sleep 5

fi

inputs:

- from: /repo

to: /io/repo

- from: /derived/debug/hail/out

to: /io/repo/hail/out

outputs:

- from: /io/repo/hail/out

to: /derived/debug/hail/out

dependsOn:

- base_image

- merge_code

- build_hail_debug_jar_and_wheel

- kind: runImage

name: build_hail_test_artifacts

image:

valueFrom: base_image.image

script: |

set -ex

cd /io/repo/hail

time tar czf test.tar.gz -C python test pytest.ini

time tar czf resources.tar.gz -C src/test resources

time tar czf data.tar.gz -C python/hail/docs data

time TESTNG_SPLITS=5 python3 generate_splits.py

time tar czf splits.tar.gz testng-splits-*.xml

inputs:

- from: /repo

to: /io/repo

outputs:

- from: /io/repo/hail/test.tar.gz

to: /test.tar.gz

- from: /io/repo/hail/resources.tar.gz

to: /resources.tar.gz

- from: /io/repo/hail/splits.tar.gz

to: /splits.tar.gz

- from: /io/repo/hail/data.tar.gz

to: /data.tar.gz

dependsOn:

- base_image

- merge_code

- kind: runImage

name: build_azure_wheel

image:

valueFrom: base_image.image

resources:

memory: standard

cpu: '2'

script: |

set -ex

cd /io/repo/hail

export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_RELEASE_MODE=1 MILLOPTS='--no-server'

time retry sh millw --no-server --version

# See `build_hail_jar_and_wheel`

time retry make shadowJar

if [ ! -f out/assembly.dest/out.jar ]; then

echo 'no out.jar found after mill assembly returned. going to sleep'

sleep 5

fi

time retry make wheel

inputs:

- from: /repo

to: /io/repo

outputs:

- from: /io/repo/hail/build/deploy/dist

to: /azure-wheel

dependsOn:

- base_image

- merge_code

- kind: buildImage2

name: hail_run_image

dockerFile: /io/repo/hail/Dockerfile.hail-run

contextPath: /io/repo

publishAs: hail-run

resources:

storage: 10Gi

cpu: "2"

memory: standard

inputs:

- from: /repo/hail/Dockerfile.hail-run

to: /io/repo/hail/Dockerfile.hail-run

- from: /repo/hail/python/pinned-requirements.txt

to: /io/repo/hail/python/pinned-requirements.txt

- from: /repo/hail/python/dev/pinned-requirements.txt

to: /io/repo/hail/python/dev/pinned-requirements.txt

- from: /repo/docker/core-site.xml

to: /io/repo/docker/core-site.xml

dependsOn:

- base_image

- merge_code

- kind: runImage

name: jvm_entryway_jar

image:

valueFrom: base_image.image

script: |

set -ex

cd /io/batch/jvm-entryway

chmod 755 ./gradlew

./gradlew shadowJar

inputs:

- from: /repo/batch

to: /io/batch

outputs:

- from: /io/batch/jvm-entryway/build/libs/jvm-entryway.jar

to: /jvm-entryway.jar

dependsOn:

- base_image

- merge_code

- kind: buildImage2

name: batch_worker_image

dockerFile: /io/repo/batch/Dockerfile.worker

contextPath: /io/repo

publishAs: batch-worker

inputs:

- from: /repo/letsencrypt/subdomains.txt

to: /io/repo/letsencrypt/subdomains.txt

- from: /repo/docker

to: /io/repo/docker

- from: /repo/batch

to: /io/repo/batch

- from: /repo/hail/python/setup-hailtop.py

to: /io/repo/hail/python/setup-hailtop.py

- from: /repo/hail/python/MANIFEST.in

to: /io/repo/hail/python/MANIFEST.in

- from: /repo/hail/python/hailtop

to: /io/repo/hail/python/hailtop

- from: /hail_version

to: /io/repo/hail/python/hailtop/hail_version

- from: /repo/gear

to: /io/repo/gear

- from: /repo/web_common

to: /io/repo/web_common

- from: /jvm-entryway.jar

to: /io/repo/batch/jvm-entryway/build/libs/jvm-entryway.jar

- from: /hail_version

to: /io/repo/hail_version

dependsOn:

- merge_code

- hail_ubuntu_image

- jvm_entryway_jar

- kind: runImage

name: upload_test_resources_to_blob_storage

image:

valueFrom: hailgenetics_hailtop_image.image

script: |

set -ex

cd /io/repo/hail/

export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

python3 -m hailtop.aiotools.copy 'null' '[

{"from": "src/test/resources",

"to": "{{ global.test_storage_uri }}/{{ token }}/test/resources"},

{"from": "python/hail/docs/data",

"to": "{{ global.test_storage_uri }}/{{ token }}/doctest/data"}

]'

secrets:

- name: test-gsa-key

namespace:

valueFrom: default_ns.name

mountPath: /test-gsa-key

inputs:

- from: /repo/hail/src/test/resources

to: /io/repo/hail/src/test/resources

- from: /repo/hail/python/hail/docs/data

to: /io/repo/hail/python/hail/docs/data

dependsOn:

- default_ns

- hailgenetics_hailtop_image

- create_test_gsa_keys

- merge_code

- kind: runImage

name: test_hail_java

numSplits: 5

image:

valueFrom: hail_run_image.image

resources:

memory: standard

cpu: '2'

script: |

set -ex

cd /io

mkdir -p src/test

tar xzf resources.tar.gz -C src/test