From 1947becbc41b6274ffb636f3d5e33eaadcc1e825 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 29 Oct 2022 19:08:45 +0200 Subject: [PATCH] build: garbage collection page Signed-off-by: CrazyMax --- _data/toc.yaml | 38 +++++---- build/building/cache/garbage-collection.md | 90 ++++++++++++++++++++++ build/building/cache/index.md | 1 + 3 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 build/building/cache/garbage-collection.md diff --git a/_data/toc.yaml b/_data/toc.yaml index 4ec03fee11..9c929eebd1 100644 --- a/_data/toc.yaml +++ b/_data/toc.yaml @@ -1521,24 +1521,28 @@ manuals: title: Kubernetes driver - path: /build/building/drivers/remote/ title: Remote driver - - path: /build/building/cache/ - title: Optimizing builds with cache - - sectiontitle: Cache backends + - sectiontitle: Cache section: - - path: /build/building/cache/backends/ - title: Overview - - path: /build/building/cache/backends/inline/ - title: Inline - - path: /build/building/cache/backends/local/ - title: Local - - path: /build/building/cache/backends/registry/ - title: Registry - - path: /build/building/cache/backends/gha/ - title: GitHub Actions - - path: /build/building/cache/backends/azblob/ - title: Azure Blob Storage - - path: /build/building/cache/backends/s3/ - title: Amazon S3 + - path: /build/building/cache/ + title: Optimizing builds with cache + - path: /build/building/cache/garbage-collection/ + title: Garbage collection + - sectiontitle: Cache backends + section: + - path: /build/building/cache/backends/ + title: Overview + - path: /build/building/cache/backends/inline/ + title: Inline + - path: /build/building/cache/backends/local/ + title: Local + - path: /build/building/cache/backends/registry/ + title: Registry + - path: /build/building/cache/backends/gha/ + title: GitHub Actions + - path: /build/building/cache/backends/azblob/ + title: Azure Blob Storage + - path: /build/building/cache/backends/s3/ + title: Amazon S3 - path: /build/building/multi-stage/ title: Multi-stage builds - path: /build/building/multi-platform/ diff --git a/build/building/cache/garbage-collection.md b/build/building/cache/garbage-collection.md new file mode 100644 index 0000000000..d6861cd495 --- /dev/null +++ b/build/building/cache/garbage-collection.md @@ -0,0 +1,90 @@ +--- +title: Garbage collection +keywords: build, buildx, buildkit, garbage collection, prune +--- + +While [`docker build prune`](../../../engine/reference/commandline/builder_prune.md) +or [`docker buildx prune`](../../../engine/reference/commandline/buildx_prune.md) +commands run at once, garbage collection runs periodically and follows an +ordered list of prune policies. + +Garbage collection runs in the BuildKit daemon. The daemon clears the build +cache when the cache size becomes too big, or when the cache age expires. The +following sections describe how you can configure both the size and age +parameters by defining garbage collection policies. + +## Configuration + +Depending on the [driver](../drivers/index.md) used by your builder instance, +the garbage collection will use a different configuration file. + +If you're using the [`docker` driver](../drivers/docker.md), garbage collection +can be configured in the [Docker Daemon configuration](../../../engine/reference/commandline/dockerd.md#daemon-configuration-file). +file: + +```json +{ + "builder": { + "gc": { + "enabled": true, + "defaultKeepStorage": "10GB", + "policy": [ + {"keepStorage": "10GB", "filter": ["unused-for=2200h"]}, + {"keepStorage": "50GB", "filter": ["unused-for=3300h"]}, + {"keepStorage": "100GB", "all": true} + ] + } + } +} +``` + +For other drivers, garbage collection can be configured using the +[BuildKit configuration](../../buildkit/toml-configuration.md) file: + +```toml +[worker.oci] + gc = true + gckeepstorage = 10000 + [[worker.oci.gcpolicy]] + keepBytes = 512000000 + keepDuration = 172800 + filters = [ "type==source.local", "type==exec.cachemount", "type==source.git.checkout"] + [[worker.oci.gcpolicy]] + all = true + keepBytes = 1024000000 +``` + +## Default policies + +Default garbage collection policies are applied to all builders if not +already set: + +``` +GC Policy rule#0: + All: false + Filters: type==source.local,type==exec.cachemount,type==source.git.checkout + Keep Duration: 48h0m0s + Keep Bytes: 512MB +GC Policy rule#1: + All: false + Keep Duration: 1440h0m0s + Keep Bytes: 26GB +GC Policy rule#2: + All: false + Keep Bytes: 26GB +GC Policy rule#3: + All: true + Keep Bytes: 26GB +``` + +* `rule#0`: if build cache uses more than 512MB delete the most easily + reproducible data after it has not been used for 2 days. +* `rule#1`: remove any data not used for 60 days. +* `rule#2`: keep the unshared build cache under cap. +* `rule#3`: if previous policies were insufficient start deleting internal data + to keep build cache under cap. + +> **Note** +> +> "Keep bytes" defaults to 10% of the size of the disk. If the disk size cannot +> be determined, it defaults to 2GB. diff --git a/build/building/cache/index.md b/build/building/cache/index.md index 6ce025c0e0..af098f5604 100644 --- a/build/building/cache/index.md +++ b/build/building/cache/index.md @@ -289,4 +289,5 @@ of continuing.) For more information on using cache to do efficient builds, see: +- [Garbage collection](garbage-collection.md) - [Cache storage backends](./backends/index.md)