From 4c36f9049ccdcca091127cc12d00b7f462fea772 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Tue, 5 Aug 2025 23:36:11 -0700 Subject: [PATCH] First pass at Operations docs Signed-off-by: Nic Cope --- content/master/_index.md | 2 + .../get-started-with-operations.md | 367 ++++++++++++ content/master/operations/_index.md | 7 + content/master/operations/cronoperation.md | 345 +++++++++++ content/master/operations/operation.md | 500 +++++++++++++++ content/master/operations/watchoperation.md | 567 ++++++++++++++++++ .../vale/styles/Crossplane/allowed-jargon.txt | 6 + .../styles/Crossplane/crossplane-words.txt | 12 + .../styles/Crossplane/spelling-exceptions.txt | 27 + 9 files changed, 1833 insertions(+) create mode 100644 content/master/get-started/get-started-with-operations.md create mode 100644 content/master/operations/_index.md create mode 100644 content/master/operations/cronoperation.md create mode 100644 content/master/operations/operation.md create mode 100644 content/master/operations/watchoperation.md diff --git a/content/master/_index.md b/content/master/_index.md index 00a6c950..7d6d54a1 100644 --- a/content/master/_index.md +++ b/content/master/_index.md @@ -23,6 +23,8 @@ Crossplane organizes its documentation into the following sections: * [Composition]({{}}) covers the key concepts of composition. +* [Operations]({{}}) covers the key concepts of operations. + * [Managed Resources]({{}}) covers the key concepts of managed resources. diff --git a/content/master/get-started/get-started-with-operations.md b/content/master/get-started/get-started-with-operations.md new file mode 100644 index 00000000..cf3d953d --- /dev/null +++ b/content/master/get-started/get-started-with-operations.md @@ -0,0 +1,367 @@ +--- +title: Get Started With Operations +weight: 300 +state: alpha +alphaVersion: 2.0 +--- + +This guide shows how to use Crossplane Operations to automate day-two +operational tasks. You create an Operation that checks SSL certificate +expiry for a website. + +**Crossplane calls this _Operations_.** Operations run function pipelines to +perform tasks that don't fit the typical resource creation pattern - like +certificate monitoring, rolling upgrades, or scheduled maintenance. + +An Operation looks like this: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: check-cert-expiry +spec: + mode: Pipeline + pipeline: + - step: check-certificate + functionRef: + name: crossplane-contrib-function-python + input: + apiVersion: python.fn.crossplane.io/v1beta1 + kind: Script + script: | + import ssl + import socket + from datetime import datetime + + from crossplane.function import request, response + + def operate(req, rsp): + hostname = "google.com" + port = 443 + + # Get SSL certificate info + context = ssl.create_default_context() + with socket.create_connection((hostname, port)) as sock: + with context.wrap_socket(sock, server_hostname=hostname) as ssock: + cert = ssock.getpeercert() + + # Parse expiration date + expiry_date = datetime.strptime(cert['notAfter'], '%b %d %H:%M:%S %Y %Z') + days_until_expiry = (expiry_date - datetime.now()).days + + # Return results in operation output + response.set_output(rsp, { + "hostname": hostname, + "certificateExpires": cert['notAfter'], + "daysUntilExpiry": days_until_expiry, + "status": "warning" if days_until_expiry < 30 else "ok" + }) +``` + + +**The Operation runs once to completion, like a Kubernetes Job.** + + +When you create the Operation, Crossplane runs the function pipeline. The +function checks SSL certificate expiry for google.com and returns the results +in the operation's output. + +This basic example shows the concept. In the walkthrough below, you create +a more realistic Operation that reads Kubernetes Ingress resources and +annotates them with certificate expiry information for monitoring tools. + +## Prerequisites + +This guide requires: + +* A Kubernetes cluster with at least 2 GB of RAM +* The Crossplane v2 preview [installed on the Kubernetes cluster]({{}}) with Operations enabled + +{{}} +Enable Operations by adding `--enable-operations` to Crossplane's startup +arguments. If using Helm: + +```shell +helm upgrade --install crossplane crossplane-stable/crossplane \ + --namespace crossplane-system \ + --set args='{"--enable-operations"}' +``` +{{}} + +## Create an operation + +Follow these steps to create your first Operation: + +1. [Create a sample Ingress](#create-a-sample-ingress) for certificate checking +1. [Install the function](#install-the-function) you want to use for the + operation +1. [Create the Operation](#create-the-operation) that checks the Ingress +1. [Check the Operation](#check-the-operation) as it runs + +### Create a sample Ingress + +Create an Ingress that references a real hostname but doesn't route actual +traffic: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: example-app + namespace: default +spec: + rules: + - host: google.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: nonexistent-service + port: + number: 80 +``` + +Save as `ingress.yaml` and apply it: + +```shell +kubectl apply -f ingress.yaml +``` + +### Grant Ingress permissions + +Operations need permission to access and change Ingresses. Create a ClusterRole +that grants Crossplane access to Ingresses: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: operations-ingress-access + labels: + rbac.crossplane.io/aggregate-to-crossplane: "true" +rules: +- apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch", "patch", "update"] +``` + +Save as `ingress-rbac.yaml` and apply it: + +```shell +kubectl apply -f ingress-rbac.yaml +``` + +### Install the function + +Operations use operation functions to implement their logic. Use the Python +function, which supports both composition and operations. + +Create this function to install Python support: + +```yaml +apiVersion: pkg.crossplane.io/v1 +kind: Function +metadata: + name: crossplane-contrib-function-python +spec: + package: xpkg.crossplane.io/crossplane-contrib/function-python:v0.2.0 +``` + +Save the function as `function.yaml` and apply it: + +```shell +kubectl apply -f function.yaml +``` + +Check that Crossplane installed the function: + +```shell {copy-lines="1"} +kubectl get -f function.yaml +NAME INSTALLED HEALTHY PACKAGE AGE +crossplane-contrib-function-python True True xpkg.crossplane.io/crossplane-contrib/function-python:v0.2.0 12s +``` + +### Create the operation + +Create this Operation that monitors the Ingress certificate: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: ingress-cert-monitor +spec: + mode: Pipeline + pipeline: + - step: check-ingress-certificate + functionRef: + name: crossplane-contrib-function-python + requirements: + requiredResources: + - requirementName: ingress + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: example-app + namespace: default + input: + apiVersion: python.fn.crossplane.io/v1beta1 + kind: Script + script: | + import ssl + import socket + from datetime import datetime + + from crossplane.function import request, response + + def operate(req, rsp): + # Get the Ingress resource + ingress = request.get_required_resource(req, "ingress") + if not ingress: + response.set_output(rsp, {"error": "No ingress resource found"}) + return + + # Extract hostname from Ingress rules + hostname = ingress["spec"]["rules"][0]["host"] + port = 443 + + # Get SSL certificate info + context = ssl.create_default_context() + with socket.create_connection((hostname, port)) as sock: + with context.wrap_socket(sock, server_hostname=hostname) as ssock: + cert = ssock.getpeercert() + + # Parse expiration date + expiry_date = datetime.strptime(cert['notAfter'], '%b %d %H:%M:%S %Y %Z') + days_until_expiry = (expiry_date - datetime.now()).days + + # Add warning if certificate expires soon + if days_until_expiry < 30: + response.warning(rsp, f"Certificate for {hostname} expires in {days_until_expiry} days") + + # Annotate the Ingress with certificate expiry info + rsp.desired.resources["ingress"].resource.update({ + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "name": ingress["metadata"]["name"], + "namespace": ingress["metadata"]["namespace"], + "annotations": { + "cert-monitor.crossplane.io/expires": cert['notAfter'], + "cert-monitor.crossplane.io/days-until-expiry": str(days_until_expiry), + "cert-monitor.crossplane.io/status": "warning" if days_until_expiry < 30 else "ok" + } + } + }) + + # Return results in operation output for monitoring + response.set_output(rsp, { + "ingressName": ingress["metadata"]["name"], + "hostname": hostname, + "certificateExpires": cert['notAfter'], + "daysUntilExpiry": days_until_expiry, + "status": "warning" if days_until_expiry < 30 else "ok" + }) +``` + + +Save the operation as `operation.yaml` and apply it: + +```shell +kubectl apply -f operation.yaml +``` + +### Check the operation + +Check that the Operation runs successfully: + +```shell {copy-lines="1"} +kubectl get -f operation.yaml +NAME SYNCED SUCCEEDED AGE +ingress-cert-monitor True True 15s +``` + +{{}} +Operations show `SUCCEEDED=True` when they complete successfully. +{{}} + +Check the Operation's detailed status: + +```shell {copy-lines="1"} +kubectl describe operation ingress-cert-monitor +# ... metadata ... +Status: + Conditions: + Last Transition Time: 2024-01-15T10:30:15Z + Reason: PipelineSuccess + Status: True + Type: Succeeded + Last Transition Time: 2024-01-15T10:30:15Z + Reason: ValidPipeline + Status: True + Type: ValidPipeline + Pipeline: + Output: + Certificate Expires: Sep 29 08:34:02 2025 GMT + Days Until Expiry: 54 + Hostname: google.com + Ingress Name: example-app + Status: ok + Step: check-ingress-certificate +``` + +{{}} +The `status.pipeline` field shows the output returned by each function step. +Use this field for tracking what the operation accomplished. +{{}} + +Check that the Operation annotated the Ingress with certificate information: + +```shell {copy-lines="1"} +kubectl get ingress example-app -o yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + cert-monitor.crossplane.io/days-until-expiry: "54" + cert-monitor.crossplane.io/expires: Sep 29 08:34:02 2025 GMT + cert-monitor.crossplane.io/status: ok + name: example-app + namespace: default +spec: + # ... ingress spec ... +``` + +{{}} +This pattern shows how Operations can both read and change existing Kubernetes +resources. The Operation annotated the Ingress with certificate expiry +information that other tools can use for monitoring and alerting. +{{}} + +## Clean up + +Delete the resources you created: + +```shell +kubectl delete -f operation.yaml +kubectl delete -f ingress.yaml +kubectl delete -f ingress-rbac.yaml +kubectl delete -f function.yaml +``` + +## Next steps + +Operations are powerful building blocks for operational workflows. Learn more +about: + +* [**Operation concepts**]({{}}) - Core + Operation features and best practices +* [**CronOperation**]({{}}) - Schedule + operations to run automatically +* [**WatchOperation**]({{}}) - Trigger + operations when resources change + +Explore the complete [Operations documentation]({{}}) for +advanced features and examples. diff --git a/content/master/operations/_index.md b/content/master/operations/_index.md new file mode 100644 index 00000000..a69146ca --- /dev/null +++ b/content/master/operations/_index.md @@ -0,0 +1,7 @@ +--- +title: Operations +weight: 52 +state: alpha +alphaVersion: v2.0-preview +description: Understand Crossplane's Operations feature +--- diff --git a/content/master/operations/cronoperation.md b/content/master/operations/cronoperation.md new file mode 100644 index 00000000..47de4c6d --- /dev/null +++ b/content/master/operations/cronoperation.md @@ -0,0 +1,345 @@ +--- +title: CronOperation +weight: 120 +state: alpha +alphaVersion: 2.0 +description: CronOperations create Operations on a schedule for recurring tasks +--- + +A `CronOperation` creates [Operations]({{}}) on a schedule, +like Kubernetes CronJobs. Use CronOperations for recurring operational tasks +such as database backups, certificate rotation, or periodic maintenance. + + +## How CronOperations work + + +CronOperations contain a template for an Operation and create new Operations +based on a cron schedule. Each scheduled run creates a new Operation that +executes once to completion. + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: CronOperation +metadata: + name: daily-backup +spec: + schedule: "0 2 * * *" # Daily at 2 AM + concurrencyPolicy: Forbid + successfulHistoryLimit: 5 + failedHistoryLimit: 3 + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: backup + functionRef: + name: function-database-backup + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: DatabaseBackupInput + retentionDays: 7 +``` + +{{}} +CronOperations are an alpha feature. You must enable Operations by adding +`--enable-operations` to Crossplane's arguments. +{{}} + +## Key features + +- **Standard cron scheduling syntax** - Uses the same format as Kubernetes CronJobs +- **Configurable concurrency policies** (Allow, Forbid, Replace) +- **Automatic cleanup of old Operations** - Maintains history limits +- **Tracks run history and running operations** - Provides visibility into scheduled runs + +## Scheduling + +CronOperations use standard cron syntax: + +``` +┌───────────── minute (0 - 59) +│ ┌───────────── hour (0 - 23) +│ │ ┌───────────── day of the month (1 - 31) +│ │ │ ┌───────────── month (1 - 12) +│ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday) +│ │ │ │ │ +│ │ │ │ │ +* * * * * +``` + +**Common schedule examples:** +- `"0 2 * * *"` - Every day at 2:00 AM +- `"0 0 * * 0"` - Every Sunday at midnight +- `"0 0 1 * *"` - Every month on the first at midnight +- `"*/15 * * * *"` - Every 15 minutes + +## Concurrency policies + +CronOperations support three concurrency policies: + +- **Allow (default)**: Multiple Operations can run simultaneously. Use this + when operations don't interfere with each other. +- **Forbid**: New Operations don't start if previous ones are still running. + Use this for operations that can't run concurrently. +- **Replace**: New Operations stop running ones before starting. Use this + when you always want the latest operation to run. + +## History management + +Control the number of completed Operations to keep: + +```yaml +spec: + successfulHistoryLimit: 5 # Keep 5 successful operations + failedHistoryLimit: 3 # Keep 3 failed operations for debugging +``` + +This helps balance debugging capabilities with resource usage. + +## Common use cases + +{{}} +The following examples use hypothetical functions for illustration. At launch, +only function-python supports operations. +{{}} + +### Scheduled database backups + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: CronOperation +metadata: + name: postgres-backup +spec: + schedule: "0 3 * * *" # Daily at 3 AM + concurrencyPolicy: Forbid # Don't allow overlapping backups + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: backup + functionRef: + name: function-postgres-backup + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: PostgresBackupInput + instance: production-db + s3Bucket: db-backups +``` + +### Scheduled maintenance + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: CronOperation +metadata: + name: weekly-maintenance +spec: + schedule: "0 3 * * 0" # Weekly on Sunday at 3 AM + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: cleanup-logs + functionRef: + name: function-log-cleanup + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: LogCleanupInput + retentionDays: 30 + - step: update-certificates + functionRef: + name: function-cert-renewal +``` + +### Periodic health checks + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: CronOperation +metadata: + name: health-check +spec: + schedule: "*/30 * * * *" # Every 30 minutes + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: check-cluster-health + functionRef: + name: function-health-check + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: HealthCheckInput + alertThreshold: 80 +``` + +## Advanced configuration + +### Complex scheduling patterns + +Advanced cron schedule examples for specific use cases: + +```yaml +# Weekdays only at 9 AM (Monday-Friday) +schedule: "0 9 * * 1-5" + +# Every 4 hours during business days +schedule: "0 8,12,16 * * 1-5" + +# First and last day of each month +schedule: "0 2 1,L * *" + +# Every quarter (1st of Jan, Apr, Jul, Oct) +schedule: "0 2 1 1,4,7,10 *" + +# Business hours only, every 2 hours +schedule: "0 9-17/2 * * 1-5" +``` + +### Starting deadline + +CronOperations support a `startingDeadlineSeconds` field that controls how +long to wait after the scheduled time before considering it too late to +create the Operation: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: CronOperation +metadata: + name: deadline-example +spec: + schedule: "0 9 * * 1-5" # Weekdays at 9 AM + startingDeadlineSeconds: 900 # 15 minutes + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: morning-tasks + functionRef: + name: function-morning-tasks +``` + +If the Operation can't start in 15 minutes of 9 AM (due to +controller downtime, resource constraints, etc.), the scheduled run is +skipped. + +Skip operations for: +- **Time-sensitive operations** - Skip operations that become meaningless if delayed +- **Resource protection** - Prevent backup Operations piling up during outages +- **SLA compliance** - Ensure operations run in acceptable time windows + +### Time zone considerations + +{{}} +CronOperations use the cluster's local time zone, same as Kubernetes CronJobs. +To ensure consistent scheduling across different environments, consider: + +1. **Standardize cluster time zones** - Use UTC in production clusters +2. **Document time zone assumptions** - Note expected time zone in comments +3. **Account for DST changes** - Be aware that some schedules may skip or repeat during transitions +{{}} + +## Status and monitoring + +CronOperations provide status information about scheduling: + +```yaml +status: + conditions: + - type: Synced + status: "True" + reason: ReconcileSuccess + - type: Scheduling + status: "True" + reason: ScheduleActive + lastScheduleTime: "2024-01-15T10:00:00Z" + lastSuccessfulTime: "2024-01-15T10:02:30Z" + runningOperationRefs: + - name: daily-backup-1705305600 +``` + +**Key status fields:** +- **Conditions**: Standard Crossplane conditions (Synced) and CronOperation-specific conditions: + - **Scheduling**: `True` when the CronOperation is actively scheduling operations, `False` when paused or has incorrect schedule syntax +- **`lastScheduleTime`**: When the CronOperation last created an Operation +- **`lastSuccessfulTime`**: When an Operation last completed successfully +- **`runningOperationRefs`**: Running Operations + +### Events + +CronOperations emit events for important activities: +- `CreateOperation` (Warning) - Scheduled operation creation failures +- `GarbageCollectOperations` (Warning) - Garbage collection failures +- `ReplaceRunningOperation` (Warning) - Running operation deletion failures +- `InvalidSchedule` (Warning) - Cron schedule parsing errors + + +### Monitoring + + + +Monitor CronOperations using: + + +```shell +# Check CronOperation status +kubectl get cronoperation my-cronop + +# View recent Operations created by the CronOperation +kubectl get operations -l crossplane.io/cronoperation=my-cronop + +# Check events +kubectl get events --field-selector involvedObject.name=my-cronop +``` + +## Best practices + +### Scheduling considerations + +1. **Consider time zones** - CronOperations use the host's local time + (same as Kubernetes CronJobs) +1. **Plan for long-running operations** - Ensure operations complete before + next scheduled run +1. **Set reasonable history limits** - Balance debugging needs with cluster + resource usage + +### Concurrency policies + +1. **Choose appropriate concurrency policies**: + - **Forbid** for backups, maintenance, or operations that must complete + alone + - **Replace** for health checks or monitoring where latest data is most + important + - **Allow** for independent tasks that can run simultaneously + +For general Operations best practices including function development and +operational considerations, see [Operation best practices]({{}}). + +## Troubleshooting + + +### CronOperation not creating Operations + + +1. Check the cron schedule syntax +1. Verify the CronOperation has `Synced=True` condition +1. Look for events indicating schedule parsing errors + +### Operations failing often + +1. Check Operation events and logs +1. Verify function capabilities include `operation` +1. Review retry limits and adjust as needed + +### Resource cleanup issues + +1. Verify you set history limits appropriately +1. Check for events about garbage collection failures + +## Next steps + +- Learn about [Operation]({{}}) for one-time operational tasks +- Learn about [WatchOperation]({{}}) for reactive operations +- [Get started with Operations]({{}}) to try scheduling your first operation \ No newline at end of file diff --git a/content/master/operations/operation.md b/content/master/operations/operation.md new file mode 100644 index 00000000..6c615b2f --- /dev/null +++ b/content/master/operations/operation.md @@ -0,0 +1,500 @@ +--- +title: Operation +weight: 110 +state: alpha +alphaVersion: 2.0 +description: Operations run function pipelines once to completion for operational tasks +--- + +An `Operation` runs a function pipeline once to completion to perform operational +tasks that don't fit the typical resource creation pattern. Unlike compositions +that continuously reconcile desired state, Operations focus on tasks like +backups, rolling upgrades, configuration validation, and scheduled maintenance. + +## How operations work + +Operations are like Kubernetes Jobs - they run once to completion rather than +continuously reconciling. Like compositions, Operations use function pipelines +to implement their logic, but they're designed for operational workflows +instead of resource composition. + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: backup-database +spec: + mode: Pipeline + pipeline: + - step: create-backup + functionRef: + name: function-database-backup + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: DatabaseBackupInput + database: production-db + retentionDays: 30 +``` + +When you create this Operation, Crossplane: + +1. **Validates** the operation and its function dependencies +2. **Executes** the function pipeline step by step +3. **Applies** any resources the functions create or change +4. **Updates** the Operation status with results and completion state + +{{}} +Operations are an alpha feature. You must enable them by adding +`--enable-operations` to Crossplane's arguments. +{{}} + +## Key characteristics + +- **Runs once to completion** (like Kubernetes Jobs) +- **Uses function pipelines** (like Compositions) +- **Can create or change any Kubernetes resources** +- **Provides detailed status and output from each step** +- **Supports retry on failure with configurable limits** + +## Operation functions vs composition functions + +Operations and compositions both use function pipelines, but with important +differences: + +**Composition Functions:** +- **Purpose**: Create and maintain resources +- **Lifecycle**: Continuous reconciliation +- **Input**: Observed composite resources +- **Output**: Desired composed resources +- **Ownership**: Creates owner references + +**Operation Functions:** +- **Purpose**: Perform operational tasks +- **Lifecycle**: Run once to completion +- **Input**: Required resources only +- **Output**: Any Kubernetes resources +- **Ownership**: Force applies without owners + +Functions can support both modes by declaring the appropriate capabilities in +their package metadata. Function authors declare this in the `crossplane.yaml` +file when building the function package: + +```yaml +apiVersion: meta.pkg.crossplane.io/v1 +kind: Function +metadata: + name: my-function +spec: + capabilities: + - composition + - operation +``` + +This allows Crossplane to know which modes the function supports and avoid +trying to use a composition-only function for operations. + +## Common use cases + +{{}} +The following examples use hypothetical functions for illustration. At launch, +only function-python supports operations. +{{}} + +### Rolling upgrades + +Use Operations for controlled rolling upgrades: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: cluster-upgrade +spec: + mode: Pipeline + pipeline: + - step: rolling-upgrade + functionRef: + name: function-cluster-upgrade + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: ClusterUpgradeInput + targetVersion: "1.28" + batches: [0.25, 0.5, 1.0] # 25%, 50%, then 100% + healthChecks: [Synced, Ready] +``` + +### One-time maintenance + +Use Operations for specific maintenance tasks: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: certificate-rotation +spec: + mode: Pipeline + pipeline: + - step: rotate-certificates + functionRef: + name: function-cert-rotation + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: CertRotationInput + targetCertificates: + matchLabels: + rotate: "true" +``` + +## Advanced configuration + +### Retry behavior + +Operations automatically retry when they fail. Configure the retry limit to +control how often attempts occur: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: resilient-operation +spec: + retryLimit: 10 # Try up to 10 times before giving up (default: 5) + mode: Pipeline + pipeline: + - step: flaky-task + functionRef: + name: function-flaky-task + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: FlakyTaskInput + # Task that might fail due to temporary issues + timeout: "30s" +``` + +**Retry behavior:** +- Each retry resets the entire pipeline - if step 2 of 3 fails, the retry + starts from step 1 +- Operations use exponential backoff: 1 s, 2 s, 4 s, 8 s, 16 s, 32 s, then 60 s + max +- Operations track the number of failures in `status.failures` +- After reaching `retryLimit`, the Operation becomes + `Succeeded=False` + +### Credentials + +Operations can provide credentials to functions through Secrets: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: secure-backup +spec: + mode: Pipeline + pipeline: + - step: backup-with-credentials + functionRef: + name: function-backup + credentials: + - name: backup-creds + source: Secret + secretRef: + namespace: crossplane-system + name: backup-credentials + key: api-key + - name: database-creds + source: Secret + secretRef: + namespace: crossplane-system + name: database-credentials + key: connection-string + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: BackupInput + destination: s3://my-backup-bucket +``` + +### Multiple pipeline steps + +Complex operations can use multiple pipeline steps: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: multi-step-deployment +spec: + mode: Pipeline + pipeline: + - step: validate-config + functionRef: + name: function-validator + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: ValidatorInput + configName: app-config + - step: backup-current + functionRef: + name: function-backup + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: BackupInput + target: current-deployment + - step: deploy-new-version + functionRef: + name: function-deploy + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: DeployInput + image: myapp:v2.0.0 + strategy: rollingUpdate + - step: verify-health + functionRef: + name: function-health-check + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: HealthCheckInput + timeout: 300s + healthEndpoint: /health +``` + +### RBAC permissions + +If your Operation needs to access resources that Crossplane doesn't have +permissions for by default, create a ClusterRole that aggregates to +Crossplane: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: operation-additional-permissions + labels: + rbac.crossplane.io/aggregate-to-crossplane: "true" +rules: +# Additional permissions for Operations +- apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "patch", "update"] +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list"] +# Add other resources your Operations need to access +``` + +This ClusterRole is automatically aggregated to Crossplane's main +ClusterRole, giving the Crossplane service account the permissions needed +for your Operations. + +{{}} +The [RBAC manager]({{}}) automatically +grants Crossplane access to Crossplane resources (MRs, XRs, etc.). You +only need to create more ClusterRoles for other Kubernetes resources +that your Operations need to access. + +For more details on RBAC configuration, see the +[Compositions RBAC documentation]({{}}). +{{}} + +### Required resources + +Operations can preload resources for functions to access: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: resource-aware-operation +spec: + mode: Pipeline + pipeline: + - step: process-deployment + functionRef: + name: function-processor + requirements: + requiredResources: + - requirementName: app-deployment + apiVersion: apps/v1 + kind: Deployment + name: my-app + namespace: production + - requirementName: app-service + apiVersion: v1 + kind: Service + name: my-app-service + namespace: production + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: ProcessorInput + action: upgrade +``` + +Functions access these resources through the standard request structure: + +```python +from crossplane.function import request, response + +def operate(req, rsp): + # Access required resources + deployment = request.get_required_resource(req, "app-deployment") + service = request.get_required_resource(req, "app-service") + + if not deployment or not service: + response.set_output(rsp, {"error": "Required resources not found"}) + return + + # Process the resources + new_replicas = deployment["spec"]["replicas"] * 2 + + # Return updated resources with full GVK and metadata for server-side apply + rsp.desired.resources["app-deployment"].resource.update({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": deployment["metadata"]["name"], + "namespace": deployment["metadata"]["namespace"] + }, + "spec": {"replicas": new_replicas} + }) +``` + +## Status and monitoring + +Operations provide rich status information: + +```yaml +status: + conditions: + - type: Synced + status: "True" + reason: ReconcileSuccess + - type: Succeeded + status: "True" + reason: PipelineSuccess + - type: ValidPipeline + status: "True" + reason: ValidPipeline + failures: 1 # Number of retry attempts + pipeline: + - step: create-backup + output: + backupId: "backup-20240115-103000" + size: "2.3GB" + appliedResourceRefs: + - apiVersion: "v1" + kind: "Secret" + namespace: "production" + name: "backup-secret" + - apiVersion: "apps/v1" + kind: "Deployment" + name: "updated-deployment" +``` + +**Key status fields:** +- **Conditions**: Standard Crossplane conditions (Synced) and Operation-specific conditions: + - **Succeeded**: `True` when the operation completed successfully, `False` when it failed + - **ValidPipeline**: `True` when all functions have the required `operation` capability +- **Failures**: Number of times the operation has failed and retried +- **Pipeline**: Output from each function step for tracking progress +- **`AppliedResourceRefs`**: References to all resources the Operation created or modified + +### Events + +Operations emit Kubernetes events for important activities: +- Function run results and warnings +- Resource apply failures +- Operation lifecycle events (creation, completion, failure) + +### Troubleshooting operations + +**Select operation status:** + +```shell +kubectl get operation my-operation -o wide +kubectl describe operation my-operation +``` + +**Common failure scenarios:** + +1. **ValidPipeline condition is False** - Function doesn't support operations: + ```yaml + conditions: + - type: ValidPipeline + status: "False" + reason: InvalidFunctionCapability + message: "Function function-name doesn't support operations" + ``` + *Solution*: use a function that declares `operation` capability. + +2. **Succeeded condition is False** - Function run failed: + ```yaml + conditions: + - type: Succeeded + status: "False" + reason: PipelineFailure + message: "Function returned error: connection timeout" + ``` + *Solution*: view function logs and fix the underlying issue. + +3. **Resource apply failures** - View events for details: + ```shell + kubectl get events --field-selector involvedObject.name=my-operation + ``` + +**Debug function runs:** + +```shell +# View function logs +kubectl logs -n crossplane-system deployment/function-python + +# Check operation events +kubectl get events --field-selector involvedObject.kind=Operation + +# Inspect operation status in detail +kubectl get operation my-operation -o jsonpath='{.status.pipeline}' | jq '.' +``` + +## Resource management + +Operations can create or change any Kubernetes resources using server-side +apply with force ownership. This means: + +**What Operations can do:** +- Create new resources of any kind +- Change existing resources by taking ownership of specific fields +- Apply changes that may conflict with other controllers + +**What Operations can't do:** +- Delete resources (current limitation of alpha implementation) +- Establish owner references (resources aren't garbage collected) +- Continuously maintain desired state (they run once) + +{{}} +Use caution with Operations that change resources managed by other controllers. +Operations force ownership when applying changes, which can cause conflicts. +{{}} + +## Best practices + +### Operation-specific practices + +1. **Plan for rollback** - Design operations to be reversible when possible, + because Operations don't auto rollback like Compositions +1. **Make operations idempotent** - Operations should be safe to retry if they + fail partway through +1. **Use required resources** - Prepopulate functions with needed resources for + efficiency rather than requesting them during running + +### Function development + +1. **Declare capabilities** - Explicitly declare `operation` capability in + function metadata to enable Operations support +1. **Return meaningful output** - Use the output field to track what the + operation accomplished for monitoring and debugging + +## Next steps + +- [Get started with Operations]({{}}) to create your first Operation +- Learn about [CronOperation]({{}}) for scheduled operations +- Learn about [WatchOperation]({{}}) for reactive operations diff --git a/content/master/operations/watchoperation.md b/content/master/operations/watchoperation.md new file mode 100644 index 00000000..6a6298b1 --- /dev/null +++ b/content/master/operations/watchoperation.md @@ -0,0 +1,567 @@ +--- +title: WatchOperation +weight: 130 +state: alpha +alphaVersion: 2.0 +description: WatchOperations create Operations when watched resources change +--- + +A `WatchOperation` creates [Operations]({{}}) when watched +Kubernetes resources change. Use WatchOperations for reactive operational +workflows such as backing up databases before deletion, validating +configurations after updates, or triggering alerts when resources fail. + + +## How WatchOperations work + + +WatchOperations watch specific Kubernetes resources and create new Operations +whenever those resources change. The changed resource is automatically injected +into the Operation for the function to process. + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: config-validator +spec: + watch: + apiVersion: v1 + kind: ConfigMap + matchLabels: + validate: "true" + concurrencyPolicy: Allow + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: validate + functionRef: + name: function-config-validator + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: ConfigValidatorInput + rules: + - required: ["database.url", "database.port"] + - format: "email" + field: "notification.email" + - step: notify + functionRef: + name: function-slack-notifier + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: SlackNotifierInput + channel: "#alerts" + severity: "warning" +``` + +{{}} +WatchOperations are an alpha feature. You must enable Operations by adding +`--enable-operations` to Crossplane's arguments. +{{}} + +## Key features + +- **Watches any Kubernetes resource type** - Not limited to Crossplane resources +- **Supports namespace and label filtering** - Target specific resources +- **Automatically injects changed resources** - Functions receive the triggering resource +- **Configurable concurrency policies** - Control operation creation + +## Resource watching + +WatchOperations can watch any Kubernetes resource with flexible filtering: + +### Watch all resources of a type + +```yaml +spec: + watch: + apiVersion: apps/v1 + kind: Deployment +``` + +### Watch resources in a specific namespace + +```yaml +spec: + watch: + apiVersion: v1 + kind: ConfigMap + namespace: production +``` + +### Watch resources with specific labels + +```yaml +spec: + watch: + apiVersion: example.org/v1 + kind: Database + matchLabels: + backup: "enabled" + environment: "production" +``` + +### Watch cluster-scoped resources + +```yaml +spec: + watch: + apiVersion: v1 + kind: Node + matchLabels: + node-role.kubernetes.io/worker: "" +``` + +## Resource injection + +When a WatchOperation creates an Operation, it automatically injects the +changed + +resource using the special requirement name +`ops.crossplane.io/watched-resource`. + +Functions can access this resource without explicitly requesting it. + +For example, when a ConfigMap with label `validate: "true"` changes, the +WatchOperation creates an Operation like this: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: Operation +metadata: + name: config-validator-abc123 +spec: + mode: Pipeline + pipeline: + - step: validate + functionRef: + name: function-config-validator + requirements: + requiredResources: + - requirementName: ops.crossplane.io/watched-resource + apiVersion: v1 + kind: ConfigMap + name: my-config + namespace: default + # ... other pipeline steps from operationTemplate +``` + +The watched resource is automatically available to functions in +`req.required_resources` under the special name +`ops.crossplane.io/watched-resource`. + +## Concurrency policies + +WatchOperations support the same concurrency policies as CronOperations: + +- **Allow (default)**: Multiple Operations can run simultaneously. Use this + when operations don't interfere with each other. +- **Forbid**: New Operations don't start if previous ones are still running. + Use this for operations that can't run concurrently. +- **Replace**: New Operations stop running ones before starting. Use this + when you always want the latest operation to run. + +## Common use cases + +{{}} +The following examples use hypothetical functions for illustration. At launch, +only function-python supports operations. +{{}} + +### Configuration validation + +Validate ConfigMaps when they change: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: config-validator +spec: + watch: + apiVersion: v1 + kind: ConfigMap + matchLabels: + validate: "true" + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: validate-config + functionRef: + name: function-config-validator + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: ConfigValidatorInput + rules: + - required: ["database.host", "database.port"] + - format: "email" + field: "notification.email" +``` + +### Database backup on deletion + +Backup databases before they're deleted: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: backup-on-deletion +spec: + watch: + apiVersion: rds.aws.crossplane.io/v1alpha1 + kind: Instance + # Note: Watching for deletion requires function logic + # to check deletion timestamp + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: create-backup + functionRef: + name: function-rds-backup + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: RDSBackupInput + retentionDays: 30 +``` + +### Resource failure alerting + +Alert when resources enter a failed state: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: failure-alerts +spec: + watch: + apiVersion: example.org/v1 + kind: App + matchLabels: + alert: "enabled" + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: check-status + functionRef: + name: function-status-checker + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: StatusCheckerInput + alertConditions: + - type: "Ready" + status: "False" + - step: send-alert + functionRef: + name: function-alertmanager + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: AlertInput + severity: "critical" +``` + +## Advanced configuration + +### Advanced watch patterns + +Complex resource watching with multiple conditions: + +```yaml +# Watch Deployments in specific namespaces with multiple label conditions +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: multi-condition-watcher +spec: + watch: + apiVersion: apps/v1 + kind: Deployment + namespace: production # Only production namespace + matchLabels: + app.kubernetes.io/managed-by: "crossplane" + environment: "prod" + backup-required: "true" + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: backup-deployment + functionRef: + name: function-deployment-backup +``` + +```yaml +# Watch custom resources across all namespaces +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: database-lifecycle-manager +spec: + watch: + apiVersion: database.example.io/v1 + kind: PostgreSQLInstance + # No namespace specified = watch all namespaces + matchLabels: + lifecycle-management: "enabled" + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: lifecycle-check + functionRef: + name: function-database-lifecycle + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: DatabaseLifecycleInput + checkDeletionTimestamp: true + autoBackup: true +``` + + +### Cross-resource workflows + +WatchOperations can watch one resource type and dynamically fetch related +resources. Here's a WatchOperation that watches Ingresses and manages +certificates: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: ingress-certificate-manager +spec: + watch: + apiVersion: networking.k8s.io/v1 + kind: Ingress + matchLabels: + auto-cert: "enabled" + operationTemplate: + spec: + mode: Pipeline + pipeline: + - step: manage-certificates + functionRef: + name: function-cert-manager + input: + apiVersion: fn.crossplane.io/v1beta1 + kind: CertManagerInput + issuer: "letsencrypt-prod" + renewBefore: "720h" # 30 days +``` + +The function examines the watched Ingress and dynamically requests related resources: + +```python +from crossplane.function import request, response + +def operate(req, rsp): + # Access the watched Ingress resource + ingress = request.get_required_resource(req, "ops.crossplane.io/watched-resource") + if not ingress: + response.set_output(rsp, {"error": "No watched resource found"}) + return + + # Extract the service name from the Ingress backend + rules = ingress.get("spec", {}).get("rules", []) + if not rules: + return + + backend = rules[0].get("http", {}).get("paths", [{}])[0].get("backend", {}) + service_name = backend.get("service", {}).get("name") + if not service_name: + return + + ingress_namespace = ingress.get("metadata", {}).get("namespace", "default") + + # Always declare what resources we need (requirements must be stable across calls) + rsp.requirements.resources["related-service"].api_version = "v1" + rsp.requirements.resources["related-service"].kind = "Service" + rsp.requirements.resources["related-service"].match_name = service_name + rsp.requirements.resources["related-service"].namespace = ingress_namespace + + # Process the service if Crossplane fetched it after our previous response + service = request.get_required_resource(req, "related-service") + if service: + create_certificate_for_service(ingress, service, rsp) +``` + +This pattern allows functions to: +1. Examine the watched resource +2. Dynamically determine what other resources you need +3. Request those resources in the next response +4. Process the more resources in next calls + +## Status and monitoring + +WatchOperations provide status information about watching: + +```yaml +status: + conditions: + - type: Synced + status: "True" + reason: ReconcileSuccess + - type: Watching + status: "True" + reason: WatchActive + watchingResources: 12 + runningOperationRefs: + - name: config-validator-anjda + - name: config-validator-f0d92 +``` + +**Key status fields:** +- **Conditions**: Standard Crossplane conditions (Synced) and WatchOperation-specific conditions: + - **Watching**: `True` when the WatchOperation is actively watching resources, `False` when paused or failed +- **`watchingResources`**: Number of resources under watch +- **`runningOperationRefs`**: Running Operations created by this WatchOperation + +### Events + +WatchOperations emit events for important activities: +- `EstablishWatched` (Warning) - Watch establishment failures +- `TerminateWatched` (Warning) - Watch termination failures +- `GarbageCollectOperations` (Warning) - Operation cleanup failures +- `CreateOperation` (Warning) - Operation creation failures +- `ReplaceRunningOperation` (Warning) - Operation replacement failures + + +### Monitoring + + + +Monitor WatchOperations using: + + +```shell +# Check WatchOperation status +kubectl get watchoperation my-watchop + +# View recent Operations created by the WatchOperation +kubectl get operations -l crossplane.io/watchoperation=my-watchop + +# Check watched resource count +kubectl describe watchoperation my-watchop + +# Check events +kubectl get events --field-selector involvedObject.name=my-watchop +``` + + +## Best practices + +### Resource selection + +1. **Use specific label selectors** - Prevent unnecessary Operations with + precise filtering +1. **Avoid high-churn resources** - Be careful watching frequently changing + resources +1. **Start small** - Begin with narrow selectors and expand as needed + +### Event handling + +1. **Implement event filtering** - Check generation, deletion timestamp, + and status conditions + to avoid processing irrelevant changes + +1. **Monitor operation volume** - Popular resources can create numerous + Operations + + +### Concurrency policies + +1. **Choose appropriate concurrency policies**: + - **Allow** for independent processing that can run in parallel + - **Forbid** for operations that must complete before processing new + changes + - **Replace** for status-checking or monitoring where only latest state + matters + +### History management + +Like CronOperations, WatchOperations automatically clean up completed Operations: + +```yaml +apiVersion: ops.crossplane.io/v1alpha1 +kind: WatchOperation +metadata: + name: config-validator +spec: + watch: + apiVersion: v1 + kind: ConfigMap + successfulHistoryLimit: 10 # Keep 10 successful Operations (default: 3) + failedHistoryLimit: 5 # Keep 5 failed Operations (default: 1) + operationTemplate: + # Operation template here +``` + +### Watched resource injection + + +WatchOperations automatically inject the changed resource into the created +Operation using a special requirement name +`ops.crossplane.io/watched-resource`: + + +```python +from crossplane.function import request, response + +def operate(req, rsp): + # Access the resource that triggered this Operation + watched_resource = request.get_required_resource(req, "ops.crossplane.io/watched-resource") + if not watched_resource: + response.set_output(rsp, {"error": "No watched resource found"}) + return + + # Process based on the watched resource + if watched_resource["kind"] == "ConfigMap": + config_data = watched_resource["data"] + # Validate configuration... +``` + +The watched resource is available in the function's `required_resources` map +without needing to declare it in the Operation template. + +For general Operations best practices including function development and +operational considerations, see [Operation best practices]({{}}). + +## Troubleshooting + + +### WatchOperation not creating Operations + + +1. Verify the WatchOperation has `Watching=True` condition +1. Check that watched resources exist and match the selector +1. Ensure resources are actually changing +1. Look for events indicating watch establishment failures + + + +### Too many Operations created + + + +1. Refine label selectors to match fewer resources +1. Consider using `Forbid` or `Replace` concurrency policy +1. Check if resources are changing more frequently than expected +1. Review function logic to ensure it's not causing resource updates + +### Operations failing to process watched resources + +1. Verify function capabilities include `operation` +1. Check that functions handle the `ops.crossplane.io/watched-resource` +1. Review function logs for processing errors +1. Ensure functions can handle the specific resource types under watch + +## Next steps + +- Learn about [Operation]({{}}) for one-time operational tasks +- Learn about [CronOperation]({{}}) for scheduled operations +- [Get started with Operations]({{}}) to create your first reactive operation \ No newline at end of file diff --git a/utils/vale/styles/Crossplane/allowed-jargon.txt b/utils/vale/styles/Crossplane/allowed-jargon.txt index 16b990f0..97602bd2 100644 --- a/utils/vale/styles/Crossplane/allowed-jargon.txt +++ b/utils/vale/styles/Crossplane/allowed-jargon.txt @@ -20,7 +20,10 @@ ClusterRole ClusterRoles command-line ConfigMap +ConfigMaps CRD +cron +CronJobs crt CSS CUE @@ -39,6 +42,7 @@ float64 GitOps Go gRPC +hostname IAM imagePullSecret init.sh @@ -64,6 +68,7 @@ OIDC PersistentVolumeClaim Pre-releases pre-releases +Prepopulate PriorityClass proselint protobuf @@ -98,6 +103,7 @@ Substrings syscall templated TLS +walkthrough tolerations UI VM diff --git a/utils/vale/styles/Crossplane/crossplane-words.txt b/utils/vale/styles/Crossplane/crossplane-words.txt index ae015516..60f2c74c 100644 --- a/utils/vale/styles/Crossplane/crossplane-words.txt +++ b/utils/vale/styles/Crossplane/crossplane-words.txt @@ -13,12 +13,17 @@ CompositeResourceDefinition CompositeResourceDefinitions CompositionRevision CompositionRevisions +composition-only config Configs CONTRIBUTING.md ControllerConfig ControllerConfigs CRDs +CronJobs +CronOperation +CronOperation-specific +CronOperations CRs Crossplane crossplane-admin @@ -45,6 +50,7 @@ function-environment-configs function-extra-resources function-go-templating function-patch-and-transform +function-python function-template-python HealthyPackageRevision Helm-like @@ -63,9 +69,11 @@ ProviderConfig ProviderConfigs ProviderRevision RunFunctionRequest +Operation-specific RunFunctionResponse Sigstore StoreConfig +SSL StoreConfigs ToCompositeFieldPath ToEnvironmentFieldPath @@ -74,6 +82,10 @@ TrimPrefix TrimSuffix UnhealthyPackageRevision UnknownPackageRevisionHealth +ValidPipeline +WatchOperation +WatchOperation-specific +WatchOperations XCluster XNetwork xpkg diff --git a/utils/vale/styles/Crossplane/spelling-exceptions.txt b/utils/vale/styles/Crossplane/spelling-exceptions.txt index 7fa28941..a88a2791 100644 --- a/utils/vale/styles/Crossplane/spelling-exceptions.txt +++ b/utils/vale/styles/Crossplane/spelling-exceptions.txt @@ -3,6 +3,7 @@ backporting built-in call-outs +google.com ClusterRoles` comma-separated conformant @@ -10,6 +11,7 @@ cross-reference Cross-resource cross-resource datastore +day-two double-check double-checks dry-run @@ -23,12 +25,16 @@ how-to in-depth in-memory left-hand +Long-running +low-traffic multi-cluster multi-region multi-tenant multi-tenancy non-empty non-Kubernetes +one-time +One-time per-element per-object per-resource @@ -57,4 +63,25 @@ user-defined v2 version-specific Job. +e.g. +high-churn +idempotency +least-privilege +long-running +Multi-step +namespace-scoped +non-production +preload +event-driven +hardcode +low-risk +Operation-level +performant +resource-intensive +status-checking +System-level +Time-sensitive +user-provided +validators +webhook-based backporting \ No newline at end of file