From 1d5fe6ad6d3ebd4c5ac0b03a3a82804270277254 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Tue, 21 Nov 2023 18:24:54 -0800 Subject: [PATCH] Add function-template-python Signed-off-by: Nic Cope --- .github/ISSUE_TEMPLATE/bug_report.md | 40 ++++++ .github/ISSUE_TEMPLATE/feature_request.md | 24 ++++ .github/PULL_REQUEST_TEMPLATE.md | 30 ++++ .github/workflows/ci.yml | 166 ++++++++++++++++++++++ Dockerfile | 39 +++++ README.md | 39 ++++- example/README.md | 25 ++++ example/composition.yaml | 17 +++ example/functions.yaml | 11 ++ example/xr.yaml | 7 + function/fn.py | 33 +++++ function/main.py | 51 +++++++ package/crossplane.yaml | 6 + pyproject.toml | 116 +++++++++++++++ tests/test_fn.py | 56 ++++++++ 15 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yml create mode 100644 Dockerfile create mode 100644 example/README.md create mode 100644 example/composition.yaml create mode 100644 example/functions.yaml create mode 100644 example/xr.yaml create mode 100644 function/fn.py create mode 100644 function/main.py create mode 100644 package/crossplane.yaml create mode 100644 pyproject.toml create mode 100644 tests/test_fn.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..d834757 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,40 @@ +--- +name: Bug Report +about: Help us diagnose and fix bugs in this Function +labels: bug +--- + + +### What happened? + + + +### How can we reproduce it? + + +### What environment did it happen in? +Function version: + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..a9ddcf0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,24 @@ +--- +name: Feature Request +about: Help us make this Function more useful +labels: enhancement +--- + + +### What problem are you facing? + + +### How could this Function help solve your problem? + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d893e4d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,30 @@ + + +### Description of your changes + + + +Fixes # + +I have: + +- [ ] Read and followed Crossplane's [contribution process]. +- [ ] Added or updated unit tests for my change. + +[contribution process]: https://git.io/fj2m9 +[docs]: https://docs.crossplane.io/contribute/contribute diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fddec8c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,166 @@ +name: CI + +on: + push: + branches: + - main + - release-* + pull_request: {} + workflow_dispatch: + inputs: + version: + description: Package version (e.g. v0.1.0) + required: false + +env: + # Common versions + PYTHON_VERSION: '3.11.5' + DOCKER_BUILDX_VERSION: 'v0.11.2' + + # These environment variables are important to the Crossplane CLI install.sh + # script. They determine what version it installs. + XP_CHANNEL: master # TODO(negz): Pin to stable once v1.14 is released. + XP_VERSION: current # TODO(negz): Pin to a version once v1.14 is released. + + # This CI job will automatically push new builds to xpkg.upbound.io if the + # XPKG_ACCESS_ID and XPKG_TOKEN secrets are set in the GitHub respository (or + # organization) settings. Create a token at https://accounts.upbound.io. + XPKG_ACCESS_ID: ${{ secrets.XPKG_ACCESS_ID }} + + # The package to push, without a version tag. The default matches GitHub. For + # example xpkg.upbound.io/crossplane/function-template-go. + XPKG: xpkg.upbound.io/${{ github.repository}} + + # The package version to push. The default is 0.0.0-gitsha. + XPKG_VERSION: ${{ inputs.version }} + +jobs: + lint: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup Hatch + run: pipx install hatch==1.7.0 + + - name: Lint + run: hatch run lint:check + + unit-test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup Hatch + run: pipx install hatch==1.7.0 + + - name: Run Unit Tests + run: hatch run test:unit + + # We want to build most packages for the amd64 and arm64 architectures. To + # speed this up we build single-platform packages in parallel. We then upload + # those packages to GitHub as a build artifact. The push job downloads those + # artifacts and pushes them as a single multi-platform package. + build: + runs-on: ubuntu-22.04 + strategy: + fail-fast: true + matrix: + arch: + - amd64 + - arm64 + steps: + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: ${{ env.DOCKER_BUILDX_VERSION }} + install: true + + - name: Checkout + uses: actions/checkout@v4 + + # We ask Docker to use GitHub Action's native caching support to speed up + # the build, per https://docs.docker.com/build/cache/backends/gha/. + - name: Build Runtime + id: image + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + target: image + build-args: + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + outputs: type=docker,dest=runtime-${{ matrix.arch }}.tar + + - name: Setup the Crossplane CLI + run: "curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh" + + - name: Build Package + run: ./crossplane xpkg build --package-file=${{ matrix.arch }}.xpkg --package-root=package/ --embed-runtime-image-tarball=runtime-${{ matrix.arch }}.tar + + - name: Upload Single-Platform Package + uses: actions/upload-artifact@v3 + with: + name: packages + path: "*.xpkg" + if-no-files-found: error + retention-days: 1 + + # This job downloads the single-platform packages built by the build job, and + # pushes them as a multi-platform package. We only push the package it the + # XPKG_ACCESS_ID and XPKG_TOKEN secrets were provided. + push: + runs-on: ubuntu-22.04 + needs: + - build + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download Single-Platform Packages + uses: actions/download-artifact@v3 + with: + name: packages + path: . + + - name: Setup the Crossplane CLI + run: "curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh" + + - name: Login to Upbound + uses: docker/login-action@v3 + if: env.XPKG_ACCESS_ID != '' + with: + registry: xpkg.upbound.io + username: ${{ secrets.XPKG_ACCESS_ID }} + password: ${{ secrets.XPKG_TOKEN }} + + # If a version wasn't explicitly passed as a workflow_dispatch input we + # default to version v0.0.0--, for example + # v0.0.0-20231101115142-1091066df799. This is a simple implementation of + # Go's pseudo-versions: https://go.dev/ref/mod#pseudo-versions. + - name: Set Default Multi-Platform Package Version + if: env.XPKG_VERSION == '' + run: echo "XPKG_VERSION=v0.0.0-$(date -d@$(git show -s --format=%ct) +%Y%m%d%H%M%S)-$(git rev-parse --short=12 HEAD)" >> $GITHUB_ENV + + - name: Push Multi-Platform Package to Upbound + if: env.XPKG_ACCESS_ID != '' + run: "./crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.XPKG }}:${{ env.XPKG_VERSION }}" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c53e71c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 + +# It's important that this is Debian 12 to match the distroless image. +FROM debian:12-slim AS build + +RUN --mount=type=cache,target=/var/lib/apt/lists \ + --mount=type=cache,target=/var/cache/apt \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + && apt-get update \ + && apt-get install --no-install-recommends --yes python3-venv git + +# Don't write .pyc bytecode files. These speed up imports when the program is +# loaded. There's no point doing that in a container where they'll never be +# persisted across restarts. +ENV PYTHONDONTWRITEBYTECODE=true + +# Use Hatch to build a wheel. The build stage must do this in a venv because +# Debian doesn't have a hatch package, and it won't let you install one globally +# using pip. +WORKDIR /build +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/pip \ + python3 -m venv /venv/build \ + && /venv/build/bin/pip install hatch \ + && /venv/build/bin/hatch build -t wheel /whl + +# Create a fresh venv and install only the function wheel into it. +RUN --mount=type=cache,target=/root/.cache/pip \ + python3 -m venv /venv/fn \ + && /venv/fn/bin/pip install /whl/*.whl + +# Copy the function venv to our runtime stage. It's important that the path be +# the same as in the build stage, to avoid shebang paths and symlinks breaking. +FROM gcr.io/distroless/python3-debian12 AS image +WORKDIR / +COPY --from=build /venv/fn /venv/fn +EXPOSE 9443 +USER nonroot:nonroot +ENTRYPOINT ["/venv/fn/bin/function"] diff --git a/README.md b/README.md index c6fc96c..a0f95dd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,39 @@ # function-template-python -A template for writing a composition function in Python +[![CI](https://github.com/crossplane/function-template-python/actions/workflows/ci.yml/badge.svg)](https://github.com/crossplane/function-template-go/actions/workflows/ci.yml) + +A template for writing a [composition function][functions] in [Python][python]. + +To learn how to use this template: + +* [Learn about how composition functions work][functions] + +If you just want to jump in and get started: + +1. Replace `function-template-python` with your function's name in + `pyproject.toml` and `package/crossplane.yaml`. +1. Add your logic to `RunFunction` in `function/fn.py` +1. Add tests for your logic in `test/test_fn.py` +1. Update this file, `README.md`, to be about your function! + +This template uses [Python][python], [Docker][docker], and the [Crossplane +CLI][cli] to build functions. + +```shell +# Lint the code - see pyproject.toml +hatch run lint:check + +# Run unit tests - see tests/test_fn.py +hatch run test:unit + +# Build the function's runtime image - see Dockerfile +$ docker build . --tag=runtime + +# Build a function package - see package/crossplane.yaml +$ crossplane xpkg build -f package --embed-runtime-image=runtime +``` + +[functions]: https://docs.crossplane.io/latest/concepts/composition-functions +[python]: https://python.io +[package docs]: https://pkg.go.dev/github.com/crossplane/function-sdk-go +[docker]: https://www.docker.com +[cli]: https://docs.crossplane.io/latest/cli \ No newline at end of file diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..25d9b5e --- /dev/null +++ b/example/README.md @@ -0,0 +1,25 @@ +# Example manifests + +You can run your function locally and test it using `crossplane beta render` +with these example manifests. + +```shell +# Run the function locally +$ hatch run python function/main.py --insecure --debug +``` + +```shell +# Then, in another terminal, call it with these example manifests +$ crossplane beta render xr.yaml composition.yaml functions.yaml -r +--- +apiVersion: example.crossplane.io/v1 +kind: XR +metadata: + name: example-xr +--- +apiVersion: render.crossplane.io/v1beta1 +kind: Result +message: I was run with input "Hello world"! +severity: SEVERITY_NORMAL +step: run-the-template +``` \ No newline at end of file diff --git a/example/composition.yaml b/example/composition.yaml new file mode 100644 index 0000000..da197e7 --- /dev/null +++ b/example/composition.yaml @@ -0,0 +1,17 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: function-template-python +spec: + compositeTypeRef: + apiVersion: example.crossplane.io/v1 + kind: XR + mode: Pipeline + pipeline: + - step: run-the-template + functionRef: + name: function-template-python + input: + apiVersion: template.fn.crossplane.io/v1beta1 + kind: Input + example: "Hello world" diff --git a/example/functions.yaml b/example/functions.yaml new file mode 100644 index 0000000..a51a108 --- /dev/null +++ b/example/functions.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: pkg.crossplane.io/v1beta1 +kind: Function +metadata: + name: function-template-python + annotations: + # This tells crossplane beta render to connect to the function locally. + render.crossplane.io/runtime: Development +spec: + # This is ignored when using the Development runtime. + package: function-template-python \ No newline at end of file diff --git a/example/xr.yaml b/example/xr.yaml new file mode 100644 index 0000000..efa87f9 --- /dev/null +++ b/example/xr.yaml @@ -0,0 +1,7 @@ +# Replace this with your XR! +apiVersion: example.crossplane.io/v1 +kind: XR +metadata: + name: example-xr +spec: + region: us-east-2 \ No newline at end of file diff --git a/function/fn.py b/function/fn.py new file mode 100644 index 0000000..ca734d6 --- /dev/null +++ b/function/fn.py @@ -0,0 +1,33 @@ +"""A Crossplane composition function.""" + +import grpc +from crossplane.function import logging, response +from crossplane.function.proto.v1beta1 import run_function_pb2 as fnv1beta1 +from crossplane.function.proto.v1beta1 import run_function_pb2_grpc as grpcv1beta1 + + +class FunctionRunner(grpcv1beta1.FunctionRunnerService): + """A FunctionRunner handles gRPC RunFunctionRequests.""" + + def __init__(self): + """Create a new FunctionRunner.""" + self.log = logging.get_logger() + + async def RunFunction( + self, req: fnv1beta1.RunFunctionRequest, _: grpc.aio.ServicerContext + ) -> fnv1beta1.RunFunctionResponse: + """Run the function.""" + log = self.log.bind(tag=req.meta.tag) + log.info("Running function") + + rsp = response.to(req) + + example = "" + if "example" in req.input: + example = req.input["example"] + + # TODO: Add your function logic here! + response.normal(rsp, f"I was run with input {example}!") + log.info("I was run!", input=example) + + return rsp diff --git a/function/main.py b/function/main.py new file mode 100644 index 0000000..7ea1197 --- /dev/null +++ b/function/main.py @@ -0,0 +1,51 @@ +"""The composition function's main CLI.""" + +import click +from crossplane.function import logging, runtime + +from function import fn + + +@click.command() +@click.option( + "--debug", + "-d", + is_flag=True, + help="Emit debug logs.", +) +@click.option( + "--address", + default="0.0.0.0:9443", + show_default=True, + help="Address at which to listen for gRPC connections", +) +@click.option( + "--tls-certs-dir", + help="Serve using mTLS certificates.", + envvar="TLS_SERVER_CERTS_DIR", +) +@click.option( + "--insecure", + is_flag=True, + help="Run without mTLS credentials. " + "If you supply this flag --tls-certs-dir will be ignored.", +) +def cli(debug: bool, address: str, tls_certs_dir: str, insecure: bool) -> None: # noqa:FBT001 # We only expect callers via the CLI. + """A Crossplane composition function.""" + try: + level = logging.Level.INFO + if debug: + level = logging.Level.DEBUG + logging.configure(level=level) + runtime.serve( + fn.FunctionRunner(), + address, + creds=runtime.load_credentials(tls_certs_dir), + insecure=insecure, + ) + except Exception as e: + click.echo(f"Cannot run function: {e}") + + +if __name__ == "__main__": + cli() diff --git a/package/crossplane.yaml b/package/crossplane.yaml new file mode 100644 index 0000000..b16e927 --- /dev/null +++ b/package/crossplane.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: meta.pkg.crossplane.io/v1beta1 +kind: Function +metadata: + name: function-template-python +spec: {} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0681ef4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,116 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "function-template-python" +description = 'A composition function' +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +keywords = [] +authors = [{ name = "Crossplane Maintainers", email = "info@crossplane.io" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.11", +] + +dependencies = [ + # TODO(negz): Push to PyPI instead. We're currently waiting on our request + # for a new PyPI org for Crossplane to be approved. Once we're using a PyPI + # package (e.g. a wheel), confirm that run_function_pb2.pyi is installed. This + # interface file makes working with RunFunctionRequest and RunFunctionResponse + # a lot easier, since it contains all the type metadata. + "function-sdk-python @ git+https://github.com/crossplane/function-sdk-python", + + # Pin at least the things we import directly. + "click==8.1.7", + "grpcio==1.59.2", + "protobuf==4.25.1", +] + +dynamic = ["version"] + +[project.urls] +Documentation = "https://github.com/crossplane/function-template-python#readme" +Issues = "https://github.com/crossplane/function-template-python/issues" +Source = "https://github.com/crossplane/function-template-python" + +[project.scripts] +function = "function.main:cli" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.envs.default] +type = "virtual" +path = ".venv-default" +dependencies = ["ipython==8.17.2"] + +[tool.hatch.envs.lint] +type = "virtual" +detached = true +path = ".venv-lint" +dependencies = ["ruff==0.1.6"] + +[tool.hatch.envs.lint.scripts] +check = "ruff check function tests && ruff format --diff function tests" + +[tool.hatch.envs.test] +type = "virtual" +path = ".venv-test" + +[tool.hatch.envs.test.scripts] +unit = "python -m unittest tests/*.py" + +[tool.ruff] +target-version = "py311" +exclude = ["function/proto/*"] +select = [ + "A", + "ARG", + "ASYNC", + "B", + "C", + "D", + "DTZ", + "E", + "EM", + "ERA", + "F", + "FBT", + "I", + "ICN", + "ISC", + "N", + "PLC", + "PLE", + "PLR", + "PLW", + "Q", + "RUF", + "S", + "T", + "TID", + "UP", + "W", + "YTT", +] +ignore = ["ISC001"] # Ruff warns this is incompatible with ruff format. + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["D"] # Don't require docstrings for tests. + +[tool.ruff.isort] +known-first-party = ["function"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.pep8-naming] +# gRPC requires this PascalCase function name. +extend-ignore-names = ["RunFunction"] diff --git a/tests/test_fn.py b/tests/test_fn.py new file mode 100644 index 0000000..fa3cd68 --- /dev/null +++ b/tests/test_fn.py @@ -0,0 +1,56 @@ +import dataclasses +import unittest + +from crossplane.function import logging, resource +from crossplane.function.proto.v1beta1 import run_function_pb2 as fnv1beta1 +from google.protobuf import duration_pb2 as durationpb +from google.protobuf import json_format +from google.protobuf import struct_pb2 as structpb + +from function import fn + + +class TestFunctionRunner(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + logging.configure(level=logging.Level.DISABLED) + + async def test_run_function(self) -> None: + @dataclasses.dataclass + class TestCase: + reason: str + req: fnv1beta1.RunFunctionRequest + want: fnv1beta1.RunFunctionResponse + + cases = [ + TestCase( + reason="The function should return the input as a result.", + req=fnv1beta1.RunFunctionRequest( + input=resource.dict_to_struct({"example": "Hello, world"}) + ), + want=fnv1beta1.RunFunctionResponse( + meta=fnv1beta1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1beta1.State(), + results=[ + fnv1beta1.Result( + severity=fnv1beta1.SEVERITY_NORMAL, + message="I was run with input Hello, world!", + ) + ], + context=structpb.Struct(), + ), + ), + ] + + runner = fn.FunctionRunner() + + for case in cases: + got = await runner.RunFunction(case.req, None) + self.assertEqual( + json_format.MessageToJson(case.want), + json_format.MessageToJson(got), + "-want, +got", + ) + + +if __name__ == "__main__": + unittest.main()