Add timeout for jobs handling (#351)

Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com>

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: 1.87.0
+          toolchain: 1.88.0
           components: clippy, rustfmt
       - name: Run clippy
         run: cargo clippy --all-targets --all-features -- --deny warnings
@@ -30,7 +30,7 @@ jobs:
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: 1.87.0
+          toolchain: 1.88.0
       - name: Run backend tests
         run: cargo test
 

Cargo.lock

@@ -411,9 +411,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.40"
+version = "4.5.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f"
+checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -421,9 +421,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.40"
+version = "4.5.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e"
+checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
 dependencies = [
  "anstream",
  "anstyle",
@@ -433,9 +433,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.40"
+version = "4.5.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce"
+checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -1222,6 +1222,17 @@ version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
 
+[[package]]
+name = "io-uring"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
+dependencies = [
+ "bitflags 2.4.2",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.7.1"
@@ -2525,17 +2536,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.45.1"
+version = "1.46.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
+checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
 dependencies = [
  "backtrace",
  "bytes",
+ "io-uring",
  "libc",
  "mio",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
+ "slab",
  "socket2 0.5.5",
  "tokio-macros",
  "windows-sys 0.52.0",

Cargo.toml

@@ -10,7 +10,7 @@ members = [
 version = "0.2.0"
 license = "Apache-2.0"
 edition = "2021"
-rust-version = "1.87"
+rust-version = "1.88"
 
 [workspace.dependencies]
 anyhow = "1.0.98"
@@ -20,7 +20,7 @@ async-trait = "0.1.88"
 axum = { version = "0.8.4", features = ["macros"] }
 base64 = "0.22.1"
 cached = { version = "0.55.1", features = ["async"] }
-clap = { version = "4.5.40", features = ["derive"] }
+clap = { version = "4.5.41", features = ["derive"] }
 clowarden-core = { path = "../clowarden-core" }
 deadpool-postgres = { version = "0.14.1", features = ["serde"] }
 figment = { version = "0.10.19", features = ["yaml", "env"] }
@@ -46,7 +46,7 @@ time = { version = "0.3.41", features = [
     "parsing",
     "serde",
 ] }
-tokio = { version = "1.45.1", features = [
+tokio = { version = "1.46.1", features = [
     "macros",
     "process",
     "rt-multi-thread",

charts/clowarden/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: clowarden
 description: CLOWarden is a tool that manages access to resources across multiple services
 type: application
-version: 0.2.1-0
+version: 0.2.1
 appVersion: 0.2.0
 kubeVersion: ">= 1.19.0-0"
 home: https://clowarden.io
@@ -42,9 +42,9 @@ annotations:
       description: Issue processing email invitations
   artifacthub.io/images: |
     - name: dbmigrator
-      image: ghcr.io/cncf/clowarden/dbmigrator:v0.2.0
+      image: ghcr.io/cncf/clowarden/dbmigrator:v0.2.1
     - name: server
-      image: ghcr.io/cncf/clowarden/server:v0.2.0
+      image: ghcr.io/cncf/clowarden/server:v0.2.1
   artifacthub.io/links: |
     - name: source
       url: https://github.com/cncf/clowarden

clowarden-server/src/jobs.rs

@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
 use tokio::{
     sync::mpsc,
     task::JoinHandle,
-    time::{self, sleep, MissedTickBehavior},
+    time::{self, sleep, timeout, MissedTickBehavior},
 };
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, instrument};
@@ -32,6 +32,9 @@ use crate::{
     tmpl,
 };
 
+/// Maximum time allowed for handling a single job (in seconds).
+const JOB_TIMEOUT: u64 = 60 * 60; // 60 minutes
+
 /// How often periodic reconcile jobs should be scheduled (in seconds).
 const RECONCILE_FREQUENCY: u64 = 60 * 60; // Every hour
 
@@ -206,9 +209,22 @@ impl OrgWorker {
 
                     // Pick next job from the queue and process it
                     Some(job) = org_jobs_rx.recv() => {
+                        let job_timeout = Duration::from_secs(JOB_TIMEOUT);
                         match job {
-                            Job::Reconcile(input) => _ = self.handle_reconcile_job(input).await,
-                            Job::Validate(input) => _ = self.handle_validate_job(input).await,
+                            Job::Reconcile(input) => {
+                                match timeout(job_timeout, self.handle_reconcile_job(input.clone())).await {
+                                    Ok(Ok(())) => {},
+                                    Ok(Err(err)) => error!(?err, ?input, "error handling reconcile job"),
+                                    Err(_) => error!(?input, "reconcile job timed out"),
+                                }
+                            }
+                            Job::Validate(input) => {
+                                match timeout(job_timeout, self.handle_validate_job(input.clone())).await {
+                                    Ok(Ok(())) => {},
+                                    Ok(Err(err)) => error!(?err, ?input, "error handling validate job"),
+                                    Err(_) => error!(?input, "validate job timed out"),
+                                }
+                            }
                         }
                     }
 

database/migrations/Dockerfile

@@ -1,5 +1,5 @@
 # Build tern
-FROM golang:1.24.4-alpine3.22 AS tern
+FROM golang:1.24.5-alpine3.22 AS tern
 RUN apk --no-cache add git
 RUN go install github.com/jackc/tern@latest