[repo] generate version.py in Makefile

.gitignore

@@ -31,7 +31,6 @@ hail/python/hail/docs/experimental/hail.experimental.DB.rst
 hail/python/hailtop/batch/docs/api/
 hail/upload-qob-jar
 hail/upload-qob-test-resources
-**/hail_version
 web_common/web_common/static/css/
 docs.tar.gz
 website/website/static/css/

Makefile

@@ -14,7 +14,7 @@ HAILGENETICS_IMAGES = $(foreach img,hail vep-grch37-85 vep-grch38-95,hailgenetic
 CI_IMAGES = ci-utils hail-buildkit base hail-run
 PRIVATE_REGISTRY_IMAGES = $(patsubst %, pushed-private-%-image, $(SPECIAL_IMAGES) $(SERVICES_PLUS_ADMIN_POD) $(CI_IMAGES) $(HAILGENETICS_IMAGES))
 
-HAILTOP_VERSION := hail/python/hailtop/hail_version
+HAILTOP_VERSION := hail/python/hailtop/version.py
 SERVICES_IMAGE_DEPS = hail-ubuntu-image $(HAILTOP_VERSION) $(shell git ls-files hail/python/hailtop gear web_common)
 
 EMPTY :=
@@ -110,7 +110,7 @@ generate-pip-lockfiles:
 	./generate-pip-lockfile.sh ci
 
 $(HAILTOP_VERSION):
-	$(MAKE) -C hail python/hailtop/hail_version
+	$(MAKE) -C hail python-version-info
 
 
 %-image: IMAGE_NAME = $(patsubst %-image,%,$@):$(TOKEN)

batch/batch/front_end/front_end.py

@@ -47,7 +47,7 @@
 from gear.database import CallError
 from gear.profiling import install_profiler_if_requested
 from gear.time_limited_max_size_cache import TimeLimitedMaxSizeCache
-from hailtop import aiotools, dictfix, httpx, uvloopx, version
+from hailtop import __version__, aiotools, dictfix, httpx, uvloopx
 from hailtop.auth import hail_credentials
 from hailtop.batch_client.globals import MAX_JOB_GROUPS_DEPTH, ROOT_JOB_GROUP_ID
 from hailtop.batch_client.parse import parse_cpu_in_mcpu, parse_memory_in_bytes, parse_storage_in_bytes
@@ -250,7 +250,7 @@ async def get_healthcheck(_) -> web.Response:
 @routes.get('/api/v1alpha/version')
 @api_security_headers
 async def rest_get_version(_) -> web.Response:
-    return web.Response(text=version())
+    return web.Response(text=__version__)
 
 
 @routes.get('/api/v1alpha/cloud')
@@ -3487,7 +3487,7 @@ async def swagger(request):
 
 @routes.get('/openapi.yaml')
 async def openapi(request):
-    page_context = {'base_path': deploy_config.base_path('batch'), 'spec_version': version()}
+    page_context = {'base_path': deploy_config.base_path('batch'), 'spec_version': __version__}
     return await render_template('batch', request, None, 'openapi.yaml', page_context)
 
 

batch/test/utils.py

@@ -4,10 +4,10 @@
 
 import hailtop.batch_client.aioclient as aiobc
 import hailtop.batch_client.client as bc
-from hailtop import pip_version
+from hailtop import __pip_version__
 
 DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04')
-HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}')
+HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}')
 
 
 @overload

build.yaml

@@ -22,16 +22,9 @@ steps:
       cd repo
       {{ code.checkout_script }}
       make -C hail python-version-info HAIL_RELEASE_MODE=1 MILLOPTS=--no-server
-      git rev-parse HEAD > git_version
     outputs:
       - from: /io/repo/
         to: /repo
-      - from: /io/repo/hail/python/hail/hail_version
-        to: /hail_version
-      - from: /io/repo/hail/python/hail/hail_pip_version
-        to: /hail_pip_version
-      - from: /io/repo/git_version
-        to: /git_version
     dependsOn:
       - git_make_bash_image
   - kind: createNamespace
@@ -279,8 +272,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
     dependsOn:
@@ -354,8 +345,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
       - from: /repo/web_common
@@ -623,8 +612,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
       - from: /repo/web_common
@@ -650,8 +637,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
       - from: /repo/web_common
@@ -673,8 +658,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail_version
     dependsOn:
       - merge_code
       - hail_ubuntu_image
@@ -692,8 +675,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
       - from: /repo/web_common
@@ -743,10 +724,8 @@ steps:
       time retry make wheel
 
       # Check wheel size is small enough for pypi (< 200 MiB)
-      HAIL_PIP_VERSION=$(cat python/hail/hail_pip_version)
-      WHEEL_PATH="build/deploy/dist/hail-$HAIL_PIP_VERSION-py3-none-any.whl"
-      du -h $WHEEL_PATH
-      $(python3 -c "import os; exit(1) if (os.path.getsize('$WHEEL_PATH')) > (200 * 1024 * 1024) else exit(0)")
+      BYTES=$(du build/deploy/dist/hail-*-py3-none-any.whl | awk '{print $1}')
+      (( BYTES < 209715200 )) || exit 1
     inputs:
       - from: /repo
         to: /io/repo
@@ -941,16 +920,12 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
       - from: /repo/web_common
         to: /io/repo/web_common
       - from: /jvm-entryway.jar
         to: /io/repo/batch/jvm-entryway/build/libs/jvm-entryway.jar
-      - from: /hail_version
-        to: /io/repo/hail_version
     dependsOn:
       - merge_code
       - hail_ubuntu_image
@@ -1354,15 +1329,13 @@ steps:
       mv python/hail/docs/_build/html www/docs/0.2
       mv python/hailtop/batch/docs/_build/html www/docs/batch
 
-      HAIL_CACHE_VERSION=$(cat python/hail/hail_version)
+      HAIL_CACHE_VERSION=$(cat env/HAIL_VERSION)
       find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +;
 
       tar czf /io/www.tar.gz www
     inputs:
-      - from: /repo/hail/python
-        to: /io/hail/python
-      - from: /hail_version
-        to: /io/repo/hail/python/hail/hail_version
+      - from: /repo/hail
+        to: /io/hail
       - from: /repo/website/website
         to: /io/website/website
       - from: /derived/release/hail/build/deploy/dist
@@ -1592,7 +1565,7 @@ steps:
       # dev deploy elides the hail-is remote, add it and retrieve the tags
       git remote add hail-is https://github.com/hail-is/hail.git
 
-      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)
+      export HAIL_PIP_VERSION=$(cat hail/env/HAIL_PIP_VERSION)
 
       if git ls-remote --exit-code --tags hail-is $HAIL_PIP_VERSION
       then
@@ -1625,8 +1598,6 @@ steps:
         to: /io/repo
       - from: /www.tar.gz
         to: /io/www.tar.gz
-      - from: /hail_pip_version
-        to: /io/hail_pip_version
     outputs:
       - from: /io/docs.tar.gz
         to: /docs.tar.gz
@@ -2435,7 +2406,7 @@ steps:
       {% else %}
       HAIL_QUERY_JAR_URL={{ global.test_storage_uri }}/{{ default_ns.name }}
       {% endif %}
-      HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/git_version).jar
+      HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/REVISION).jar
 
       python3 -m hailtop.aiotools.copy 'null' '[{"from": "/io/hail.jar", "to": "'${HAIL_QUERY_JAR_URL}'"}]'
     secrets:
@@ -2446,8 +2417,8 @@ steps:
     inputs:
       - from: /derived/release/hail/out/assembly.dest/out.jar
         to: /io/hail.jar
-      - from: /git_version
-        to: /io/git_version
+      - from: /repo/hail/env/REVISION
+        to: /io/REVISION
     dependsOn:
       - default_ns
       - deploy_batch
@@ -2840,7 +2811,6 @@ steps:
       cp -R /io/repo/hail/python ./hail/
       cp /io/repo/hail/Makefile ./hail/
       cp /io/repo/hail/env_var.mk ./hail/
-      cp /io/repo/hail/version.mk ./hail/
       cp -R /io/repo/web_common ./
       cp /io/repo/hail/python/setup-hailtop.py ./hail/python/
 
@@ -3276,7 +3246,7 @@ steps:
       gcloud config set project hail-vdc
       gcloud config set dataproc/region us-central1
 
-      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
+      if git ls-remote --exit-code --tags origin $(cat hail/env/HAIL_PIP_VERSION)
       then
           echo "tag $HAIL_PIP_VERSION already exists"
           exit 0
@@ -3291,8 +3261,6 @@ steps:
       - merge_code
       - build_hail_jar_and_wheel
     inputs:
-      - from: /hail_pip_version
-        to: /io/hail_pip_version
       - from: /repo
         to: /io/repo
       - from: /derived/release/hail/build/deploy/dist
@@ -3322,7 +3290,7 @@ steps:
       gcloud config set project hail-vdc
       gcloud config set dataproc/region us-central1
 
-      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
+      if git ls-remote --exit-code --tags origin $(cat hail/env/HAIL_PIP_VERSION)
       then
           echo "tag $HAIL_PIP_VERSION already exists"
           exit 0
@@ -3337,8 +3305,6 @@ steps:
       - merge_code
       - build_hail_jar_and_wheel
     inputs:
-      - from: /hail_pip_version
-        to: /io/hail_pip_version
       - from: /repo
         to: /io/repo
       - from: /derived/release/hail/build/deploy/dist
@@ -3375,7 +3341,7 @@ steps:
       export GIT_ASKPASS=/io/git-askpass
 
       cd /io/repo/hail
-      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)
+      export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION)
 
       if git ls-remote --exit-code --tags origin $HAIL_PIP_VERSION
       then
@@ -3391,9 +3357,9 @@ steps:
 
       echo Setting arguments and invoking release.sh.
 
-      HAIL_VERSION=$(cat /io/hail_version) \
+      HAIL_VERSION=$(cat env/HAIL_VERSION) \
       WHEEL="build/deploy/dist/hail-${HAIL_PIP_VERSION}-py3-none-any.whl" \
-      GIT_VERSION=$(cat /io/git_version) \
+      GIT_VERSION=$(cat env/REVISION) \
       REMOTE=origin \
       GITHUB_OAUTH_HEADER_FILE=/io/github-oauth \
       HAIL_GENETICS_HAIL_IMAGE=docker://{{ hailgenetics_hail_image.image }} \
@@ -3408,12 +3374,6 @@ steps:
 
       echo '1' > /io/release-hail-flag
     inputs:
-      - from: /hail_version
-        to: /io/hail_version
-      - from: /hail_pip_version
-        to: /io/hail_pip_version
-      - from: /git_version
-        to: /io/git_version
       - from: /repo
         to: /io/repo
       - from: /derived/release/hail/out/assembly.dest/out.jar
@@ -3478,8 +3438,6 @@ steps:
       - ci_utils_image
       - release
     inputs:
-      - from: /hail_pip_version
-        to: /io/hail_pip_version
       - from: /release-hail-flag
         to: /io/release-hail-flag
       - from: /repo
@@ -3502,18 +3460,16 @@ steps:
           exit 0
       fi
 
-      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)
-      export GITHUB_OAUTH_HEADER_FILE=/io/github-oauth
-      cd /io
-
       printf 'Authorization: token ' > github-oauth
-      cat /hail-ci-0-1-github-oauth-token/oauth-token >>github-oauth
+      cat /hail-ci-0-1-github-oauth-token/oauth-token >> /io/github-oauth
       printf '#!/bin/bash\necho ' > git-askpass
-      cat /hail-ci-0-1-github-oauth-token/oauth-token >>git-askpass
-      chmod 755 git-askpass
+      cat /hail-ci-0-1-github-oauth-token/oauth-token >> /io/git-askpass
+      chmod 755 /io/git-askpass
       export GIT_ASKPASS=/io/git-askpass
 
       cd /io/repo/hail
+      export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION)
+      export GITHUB_OAUTH_HEADER_FILE=/io/github-oauth
       bash scripts/make-terra-prs.sh
   - kind: runImage
     name: mirror_hailgenetics_images
@@ -3537,12 +3493,12 @@ steps:
       set -x
 
       cd /io/docker/hailgenetics
-      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)
+      export HAIL_PIP_VERSION=$(cat /io/HAIL_PIP_VERSION)
       export DOCKER_PREFIX={{ global.docker_prefix }}
       bash mirror_images.sh
     inputs:
-      - from: /hail_pip_version
-        to: /io/hail_pip_version
+      - from: /repo/hail/env/HAIL_PIP_VERSION
+        to: /io/HAIL_PIP_VERSION
       - from: /repo/docker/hailgenetics/mirror_images.sh
         to: /io/docker/hailgenetics/mirror_images.sh
       - from: /repo/docker/copy_image.sh
@@ -3569,8 +3525,6 @@ steps:
         to: /io/repo/hail/python/MANIFEST.in
       - from: /repo/hail/python/hailtop
         to: /io/repo/hail/python/hailtop
-      - from: /hail_version
-        to: /io/repo/hail/python/hailtop/hail_version
       - from: /repo/gear
         to: /io/repo/gear
       - from: /repo/web_common
@@ -3743,8 +3697,6 @@ steps:
         to: /io/repo/hail/python
       - from: /repo/hail/scripts/benchmark_in_batch.py
         to: /io/repo/hail/scripts/benchmark_in_batch.py
-      - from: /hail_version
-        to: /io/hail_version
       - from: /release-hail-flag
         to: /io/release-hail-flag
     script: |
@@ -3760,7 +3712,7 @@ steps:
       pushd /io/repo/hail
 
       BUCKET=hail-benchmarks-2
-      HAIL_VERSION=$(cat /io/hail_version)
+      HAIL_VERSION=$(cat env/HAIL_VERSION)
       RESULTS_PREFIX="gs://${BUCKET}/{{ scope }}/${HAIL_VERSION}"
 
       export HAIL_BATCH_REGIONS={{ global.gcp_region }}