feat(async-upload): generate requirements.txt and VCS (#1502)

* feat(async-upload): generate requirements.txt and VCS generate requirements.txt as part of 'poetry install' helpful for hermetic builds of the Container image too place the requirements.txt in VCS git repo ensure with ci/GHA that the file committed is up-to-date, like for other ci/GHA already in place Signed-off-by: Matteo Mortari <matteo.mortari@gmail.com> * real requirements.txt Signed-off-by: Matteo Mortari <matteo.mortari@gmail.com> --------- Signed-off-by: Matteo Mortari <matteo.mortari@gmail.com>
2025-08-20 19:05:06 +02:00 · 2025-08-20 19:05:06 +02:00 · 4236e064cf
parent 83f1140737
commit 4236e064cf
4 changed files with 82 additions and 14 deletions
--- a/.github/workflows/async-upload-test.yml
+++ b/.github/workflows/async-upload-test.yml
@ -43,7 +43,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: "3.10" # refers to the Container image
+          python-version: "3.11" # refers to the Container image
      - name: Install Poetry
        run: |
          pipx install poetry
@ -70,7 +70,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: "3.10" # refers to the Container image
+          python-version: "3.11" # refers to the Container image
      - name: Install Poetry
        run: |
          pipx install poetry
@ -81,3 +81,34 @@ jobs:
      - name: Execute Sample Job E2E test
        run: |
          make test-integration
+  check-autogen:
+    name: Check generated code or files are in sync
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: jobs/async-upload
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install Poetry
+        run: |
+          pipx install poetry
+      - name: Install dependencies
+        run: |
+          make install
+      - name: Check if there are uncommitted file changes
+        run: |
+          clean=$(git status --porcelain)
+          if [[ -z "$clean" ]]; then
+              echo "Empty git status --porcelain: $clean"
+          else
+              echo "Uncommitted file changes detected: $clean"
+              git diff
+              exit 1
+          fi
--- a/jobs/async-upload/Dockerfile
+++ b/jobs/async-upload/Dockerfile
@ -14,22 +14,13 @@ RUN microdnf update -y && \
 # security/env hardening
 ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
-    POETRY_VERSION=2.1.3 \
-    POETRY_HOME="/opt/poetry" \
    PIP_NO_CACHE_DIR=off \
    PIP_DISABLE_PIP_VERSION_CHECK=on

-RUN python -m pip install pipx
-RUN pipx install poetry
-ENV PATH="/root/.local/bin:$PATH"
-RUN poetry --version
-
 WORKDIR /app
-COPY ./poetry.lock ./pyproject.toml ./
+COPY ./poetry.lock ./pyproject.toml ./requirements.txt ./

-RUN poetry self add poetry-plugin-export \
-    && poetry export --format requirements.txt --output requirements.txt --without-hashes \
-    && python -m pip install -r requirements.txt \
+RUN python -m pip install -r requirements.txt \
    && echo "Installation completed" \
    && echo "Python path:" && python -c "import sys; print('\\n'.join(sys.path))" \
    && echo "Installed packages:" \
--- a/jobs/async-upload/Makefile
+++ b/jobs/async-upload/Makefile
@ -92,5 +92,11 @@ test-integration-run:
 	CONTAINER_IMAGE_URI=$(JOB_IMG) poetry run pytest --integration tests/integration/ -vs

 .PHONY: install
-install:
+install: install-poetry-export
 	poetry install
+	poetry export --format requirements.txt --without-hashes --output requirements.txt
+	@echo "# This file is automatically generated by poetry export. Do not edit manually." | cat - requirements.txt > requirements.txt.tmp && mv requirements.txt.tmp requirements.txt
+
+.PHONY: install-poetry-export
+install-poetry-export:
+	poetry self add poetry-plugin-export
--- a/jobs/async-upload/requirements.txt
+++ b/jobs/async-upload/requirements.txt
@ -0,0 +1,40 @@
+# This file is automatically generated by poetry export. Do not edit manually.
+aiohappyeyeballs==2.6.1 ; python_version >= "3.11" and python_version < "4.0"
+aiohttp-retry==2.9.1 ; python_version >= "3.11" and python_version < "4.0"
+aiohttp==3.12.15 ; python_version >= "3.11" and python_version < "4.0"
+aiosignal==1.4.0 ; python_version >= "3.11" and python_version < "4.0"
+annotated-types==0.7.0 ; python_version >= "3.11" and python_version < "4.0"
+attrs==25.3.0 ; python_version >= "3.11" and python_version < "4.0"
+boto3==1.39.4 ; python_version >= "3.11" and python_version < "4.0"
+botocore==1.39.4 ; python_version >= "3.11" and python_version < "4.0"
+certifi==2025.7.14 ; python_version >= "3.11" and python_version < "4.0"
+charset-normalizer==3.4.2 ; python_version >= "3.11" and python_version < "4.0"
+click==8.2.1 ; python_version >= "3.11" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows"
+configargparse==1.7.1 ; python_version >= "3.11" and python_version < "4.0"
+eval-type-backport==0.2.2 ; python_version >= "3.11" and python_version < "4.0"
+filelock==3.18.0 ; python_version >= "3.11" and python_version < "4.0"
+frozenlist==1.7.0 ; python_version >= "3.11" and python_version < "4.0"
+fsspec==2025.7.0 ; python_version >= "3.11" and python_version < "4.0"
+hf-xet==1.1.5 ; python_version >= "3.11" and python_version < "4.0" and (platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "arm64" or platform_machine == "aarch64")
+huggingface-hub==0.34.4 ; python_version >= "3.11" and python_version < "4.0"
+idna==3.10 ; python_version >= "3.11" and python_version < "4.0"
+jmespath==1.0.1 ; python_version >= "3.11" and python_version < "4.0"
+model-registry==0.2.19 ; python_version >= "3.11" and python_version < "4.0"
+multidict==6.4.4 ; python_version >= "3.11" and python_version < "4.0"
+nest-asyncio==1.6.0 ; python_version >= "3.11" and python_version < "4.0"
+olot==0.1.8 ; python_version >= "3.11" and python_version < "4.0"
+packaging==25.0 ; python_version >= "3.11" and python_version < "4.0"
+propcache==0.3.2 ; python_version >= "3.11" and python_version < "4.0"
+pydantic-core==2.33.2 ; python_version >= "3.11" and python_version < "4.0"
+pydantic==2.11.5 ; python_version >= "3.11" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "4.0"
+pyyaml==6.0.2 ; python_version >= "3.11" and python_version < "4.0"
+requests==2.32.4 ; python_version >= "3.11" and python_version < "4.0"
+s3transfer==0.13.0 ; python_version >= "3.11" and python_version < "4.0"
+six==1.17.0 ; python_version >= "3.11" and python_version < "4.0"
+tqdm==4.67.1 ; python_version >= "3.11" and python_version < "4.0"
+typing-extensions==4.13.2 ; python_version >= "3.11" and python_version < "4.0"
+typing-inspection==0.4.1 ; python_version >= "3.11" and python_version < "4.0"
+urllib3==2.5.0 ; python_version >= "3.11" and python_version < "4.0"
+yarl==1.20.1 ; python_version >= "3.11" and python_version < "4.0"