Update vendor containers/(common,storage,buildah,image)

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
This commit is contained in:
Daniel J Walsh 2022-10-27 16:26:57 -04:00
parent 26e5661c27
commit 6fe64591d6
No known key found for this signature in database
GPG Key ID: A2DF901DABE2C028
283 changed files with 32861 additions and 4204 deletions

View File

@ -283,7 +283,7 @@ build-all-new-commits:
.PHONY: vendor
vendor:
$(GO) mod tidy
$(GO) mod tidy -compat=1.17
$(GO) mod vendor
$(GO) mod verify

47
go.mod
View File

@ -11,18 +11,18 @@ require (
github.com/container-orchestrated-devices/container-device-interface v0.5.2
github.com/containernetworking/cni v1.1.2
github.com/containernetworking/plugins v1.1.1
github.com/containers/buildah v1.28.0
github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c
github.com/containers/buildah v1.28.1-0.20221028091010-9f7a94f34a74
github.com/containers/common v0.50.2-0.20221027091852-54845c3325e3
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67
github.com/containers/image/v5 v5.23.1-0.20221027111947-5f02f763c564
github.com/containers/ocicrypt v1.1.6
github.com/containers/psgo v1.8.0
github.com/containers/storage v1.43.1-0.20221018232308-33c396d1aacf
github.com/containers/storage v1.43.1-0.20221027113344-232bf398bd68
github.com/coreos/go-systemd/v22 v22.4.0
github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3
github.com/cyphar/filepath-securejoin v0.2.3
github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001
github.com/docker/docker v20.10.20+incompatible
github.com/docker/docker v20.10.21+incompatible
github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11
github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651
github.com/docker/go-units v0.5.0
@ -41,7 +41,7 @@ require (
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
github.com/nxadm/tail v1.4.8
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.22.1
github.com/onsi/gomega v1.23.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc2
github.com/opencontainers/runc v1.1.4
@ -51,7 +51,7 @@ require (
github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df
github.com/rootless-containers/rootlesskit v1.0.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.6.0
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.1
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
@ -60,9 +60,9 @@ require (
github.com/vbauerster/mpb/v7 v7.5.3
github.com/vishvananda/netlink v1.1.1-0.20220115184804-dd687eb2f2d4
go.etcd.io/bbolt v1.3.6
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0
golang.org/x/sys v0.1.0
golang.org/x/term v0.1.0
golang.org/x/text v0.4.0
google.golang.org/protobuf v1.28.1
gopkg.in/inf.v0 v0.9.1
@ -71,13 +71,13 @@ require (
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Microsoft/hcsshim v0.9.4 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/chzyer/readline v1.5.1 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/containerd/containerd v1.6.8 // indirect
github.com/containerd/containerd v1.6.9 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
@ -87,7 +87,7 @@ require (
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsouza/go-dockerclient v1.8.3 // indirect
github.com/fsouza/go-dockerclient v1.9.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@ -102,7 +102,7 @@ require (
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/letsencrypt/boulder v0.0.0-20220723181115-27de4befb95e // indirect
github.com/letsencrypt/boulder v0.0.0-20220929215747-76583552c2be // indirect
github.com/manifoldco/promptui v0.9.0 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
@ -118,27 +118,26 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/proglottis/gpgme v0.1.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/seccomp/libseccomp-golang v0.10.0 // indirect
github.com/sigstore/sigstore v1.4.2 // indirect
github.com/sigstore/sigstore v1.4.4 // indirect
github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
github.com/sylabs/sif/v2 v2.8.0 // indirect
github.com/sylabs/sif/v2 v2.8.1 // indirect
github.com/tchap/go-patricia v2.3.0+incompatible // indirect
github.com/theupdateframework/go-tuf v0.5.1 // indirect
github.com/theupdateframework/go-tuf v0.5.2-0.20220930112810-3890c1e7ace4 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 // indirect
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.1.0 // indirect
golang.org/x/tools v0.1.12 // indirect
google.golang.org/genproto v0.0.0-20220720214146-176da50484ac // indirect
google.golang.org/grpc v1.48.0 // indirect
google.golang.org/genproto v0.0.0-20220929141241-1ce7b20da813 // indirect
google.golang.org/grpc v1.49.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
@ -146,5 +145,3 @@ require (
)
replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc
replace github.com/containers/storage => github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3

1260
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -547,9 +547,10 @@ func makeRuntime(runtime *Runtime) (retErr error) {
// This ensures that no two processes will be in runtime.refresh at once
aliveLock.Lock()
doRefresh := false
unLockFunc := aliveLock.Unlock
defer func() {
if aliveLock.Locked() {
aliveLock.Unlock()
if unLockFunc != nil {
unLockFunc()
}
}()
@ -568,7 +569,8 @@ func makeRuntime(runtime *Runtime) (retErr error) {
logrus.Debug("Invalid systemd user session for current user")
}
}
aliveLock.Unlock() // Unlock to avoid deadlock as BecomeRootInUserNS will reexec.
unLockFunc()
unLockFunc = nil
pausePid, err := util.GetRootlessPauseProcessPidPathGivenDir(runtime.config.Engine.TmpDir)
if err != nil {
return fmt.Errorf("could not get pause process pid file path: %w", err)

View File

@ -40,9 +40,7 @@ func TryJoinPauseProcess(pausePidPath string) (bool, int, error) {
pidFileLock.Lock()
defer func() {
if pidFileLock.Locked() {
pidFileLock.Unlock()
}
pidFileLock.Unlock()
}()
// Now the pause PID file is locked. Try to join once again in case it changed while it was not locked.

View File

@ -1,6 +1,6 @@
From d22e44c8fb1c87afb90391188733f7ce8fea005d Mon Sep 17 00:00:00 2001
From 0e31bb3a189ecd3e3fca36d16bb29b5162cb7d5c Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Tue, 9 Feb 2021 17:28:05 -0700
Date: Thu, 6 Oct 2022 17:32:59 -0600
Subject: [PATCH] tweaks for running buildah tests under podman
Signed-off-by: Ed Santiago <santiago@redhat.com>
@ -9,10 +9,10 @@ Signed-off-by: Ed Santiago <santiago@redhat.com>
1 file changed, 67 insertions(+), 3 deletions(-)
diff --git a/tests/helpers.bash b/tests/helpers.bash
index 8cb93c3a..72c1c797 100644
index aab3f72e..fe02a6f1 100644
--- a/tests/helpers.bash
+++ b/tests/helpers.bash
@@ -52,6 +52,23 @@ EOF
@@ -70,6 +70,23 @@ EOF
ROOTDIR_OPTS="--root ${TEST_SCRATCH_DIR}/root --runroot ${TEST_SCRATCH_DIR}/runroot --storage-driver ${STORAGE_DRIVER}"
BUILDAH_REGISTRY_OPTS="--registries-conf ${TEST_SOURCES}/registries.conf --registries-conf-dir ${TEST_SCRATCH_DIR}/registries.d --short-name-alias-conf ${TEST_SCRATCH_DIR}/cache/shortnames.conf"
PODMAN_REGISTRY_OPTS="--registries-conf ${TEST_SOURCES}/registries.conf"
@ -36,7 +36,7 @@ index 8cb93c3a..72c1c797 100644
}
function starthttpd() {
@@ -95,6 +112,12 @@ function teardown_tests() {
@@ -113,6 +130,12 @@ function teardown_tests() {
stop_git_daemon
stop_registry
@ -49,7 +49,7 @@ index 8cb93c3a..72c1c797 100644
# Workaround for #1991 - buildah + overlayfs leaks mount points.
# Many tests leave behind /var/tmp/.../root/overlay and sub-mounts;
# let's find those and clean them up, otherwise 'rm -rf' fails.
@@ -187,6 +210,10 @@ function podman() {
@@ -205,6 +228,10 @@ function podman() {
command ${PODMAN_BINARY:-podman} ${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
}
@ -60,7 +60,7 @@ index 8cb93c3a..72c1c797 100644
# There are various scenarios where we would like to execute `tests` as rootless user, however certain commands like `buildah mount`
# do not work in rootless session since a normal user cannot mount a filesystem unless they're in a user namespace along with its
# own mount namespace. In order to run such specific commands from a rootless session we must perform `buildah unshare`.
@@ -248,8 +275,36 @@ function run_buildah() {
@@ -266,8 +293,36 @@ function run_buildah() {
--retry) retry=3; shift;; # retry network flakes
esac
@ -98,18 +98,18 @@ index 8cb93c3a..72c1c797 100644
# If session is rootless and `buildah mount` is invoked, perform unshare,
# since normal user cannot mount a filesystem unless they're in a user namespace along with its own mount namespace.
@@ -263,8 +318,8 @@ function run_buildah() {
@@ -281,8 +336,8 @@ function run_buildah() {
retry=$(( retry - 1 ))
# stdout is only emitted upon error; this echo is to help a debugger
- echo "\$ $BUILDAH_BINARY $*"
- echo "${_LOG_PROMPT} $BUILDAH_BINARY $*"
- run env CONTAINERS_CONF=${CONTAINERS_CONF:-$(dirname ${BASH_SOURCE})/containers.conf} timeout --foreground --kill=10 $BUILDAH_TIMEOUT ${BUILDAH_BINARY} ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
+ echo "\$ $cmd_basename $*"
+ echo "${_LOG_PROMPT} $cmd_basename $*"
+ run env CONTAINERS_CONF=${CONTAINERS_CONF:-$(dirname ${BASH_SOURCE})/containers.conf} timeout --foreground --kill=10 $BUILDAH_TIMEOUT ${podman_or_buildah} ${_opts} "$@"
# without "quotes", multiple lines are glommed together into one
if [ -n "$output" ]; then
echo "$output"
@@ -596,6 +651,15 @@ function skip_if_no_docker() {
@@ -614,6 +669,15 @@ function skip_if_no_docker() {
fi
}
@ -126,5 +126,5 @@ index 8cb93c3a..72c1c797 100644
daemondir=${TEST_SCRATCH_DIR}/git-daemon
mkdir -p ${daemondir}/repo
--
2.36.1
2.37.3

View File

@ -18,9 +18,15 @@ usage="Usage: $ME [--help] [--no-checkout] [--no-test] [--filter=TESTNAME]
Flags, useful for manual debugging:
--no-checkout Skip checkout step, go directly to running tests
--no-test Do checkout only, but do not run tests
--no-checkout Skip checkout step, go directly to running tests.
Useful when rerunning tests.
--no-test Do checkout only, but do not run tests. Useful when
testing the apply-podman-deltas script.
--filter=NAME Passed on to bats; runs only tests that match NAME
--remote Test with podman-remote. Will start its own server.
"
# Parse command-line options (used in development only, not in CI)
@ -33,7 +39,8 @@ for i; do
--no-checkout) do_checkout= ; shift;;
--no-test) do_test= ; shift;;
--filter=*) bats_filter=("--filter" "$value"); shift;;
-h|--help) echo "$usage"; exit 0;;
--remote) PODBIN_NAME=remote;;
-h|--help) echo "$usage"; exit 0;;
*) echo "$ME: Unrecognized option '$i'" >&2; exit 1;;
esac
done
@ -183,19 +190,7 @@ You will want to:
"
(set -x;git am --reject <$PATCHES)
# FIXME FIXME FIXME! Remove this once buildah 4377 has made its way
# into this podman branch.
if ! grep -q SAFEIMAGE tests/helpers.bash; then
failhint="
You are vendoring a new buildah. Please edit $0
and remove the block between the FIXMEs.
Also remove safeimage-patch.diff.
"
(set -x;git am --reject <${BUD_TEST_DIR}/safeimage-patch.diff)
fi
# FIXME FIXME FIXME! Remove down to here.
# Now apply our custom skips and error-message changes. This is maintained
# Now apply our custom skips and error-message changes. This is maintained
# in a custom script, not a .diff file, because diffs are WAY too hard for
# humans to read and update.
APPLY=apply-podman-deltas

View File

@ -1,85 +0,0 @@
From 7aa34b86fb2e6282f29f78e64537f5f2683d605f Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Wed, 26 Oct 2022 05:58:14 -0600
Subject: [PATCH] Define and use a safe, reliable test image
The ubi8 image changed some architecture string yesterday and
broke all our CI on all branches, including podman because
it too runs bud.bats tests.
Solution: use a safe, reliable, trustworthy test image (the
one built and used for podman). Because that image does not
have the same labels, confirm pull using .Architecture
instead.
We should stop using ubi8 and registry.redhat (#4318). They
are unreliable. This PR doesn't fix that; it's just an
emergency patch for one specific CI break. We can use
this as a basis for future removals of ubi8.
Signed-off-by: Ed Santiago <santiago@redhat.com>
---
tests/bud.bats | 22 +++++++++++-----------
tests/helpers.bash | 7 +++++++
2 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/tests/bud.bats b/tests/bud.bats
index 78b4e133..c347fff2 100644
--- a/tests/bud.bats
+++ b/tests/bud.bats
@@ -4227,24 +4227,24 @@ _EOF
mytmpdir=${TEST_SCRATCH_DIR}/my-dir
mkdir -p ${mytmpdir}
cat > $mytmpdir/Containerfile << _EOF
-FROM registry.access.redhat.com/ubi8-minimal
+FROM $SAFEIMAGE
_EOF
- run_buildah build -f Containerfile --pull=false -q --arch=amd64 -t image-amd $WITH_POLICY_JSON ${mytmpdir}
- run_buildah inspect --format '{{ index .Docker.Config.Labels "architecture" }}' image-amd
- expect_output --substring x86_64
+ run_buildah build --pull=false -q --arch=amd64 -t image-amd $WITH_POLICY_JSON ${mytmpdir}
+ run_buildah inspect --format '{{ .OCIv1.Architecture }}' image-amd
+ expect_output amd64
- # Tag the image to localhost/ubi8-minimal to make sure that the image gets
+ # Tag the image to localhost/safeimage to make sure that the image gets
# pulled since the local one does not match the requested architecture.
- run_buildah tag image-amd localhost/ubi8-minimal
- run_buildah build -f Containerfile --pull=false -q --arch=arm64 -t image-arm $WITH_POLICY_JSON ${mytmpdir}
- run_buildah inspect --format '{{ index .Docker.Config.Labels "architecture" }}' image-arm
- expect_output --substring arm64
+ run_buildah tag image-amd localhost/${SAFEIMAGE_NAME}:${SAFEIMAGE_TAG}
+ run_buildah build --pull=false -q --arch=arm64 -t image-arm $WITH_POLICY_JSON ${mytmpdir}
+ run_buildah inspect --format '{{ .OCIv1.Architecture }}' image-arm
+ expect_output arm64
run_buildah inspect --format '{{ .FromImageID }}' image-arm
fromiid=$output
- run_buildah inspect --format '{{ index .OCIv1.Architecture }}' $fromiid
- expect_output --substring arm64
+ run_buildah inspect --format '{{ .OCIv1.Architecture }}' $fromiid
+ expect_output arm64
}
@test "bud --file with directory" {
diff --git a/tests/helpers.bash b/tests/helpers.bash
index f415d534..aab3f72e 100644
--- a/tests/helpers.bash
+++ b/tests/helpers.bash
@@ -13,5 +13,12 @@ OCI=$(${BUILDAH_BINARY} info --format '{{.host.OCIRuntime}}' || command -v runc
# Default timeout for a buildah command.
BUILDAH_TIMEOUT=${BUILDAH_TIMEOUT:-300}
+# Safe reliable unchanging test image
+SAFEIMAGE_REGISTRY=${SAFEIMAGE_REGISTRY:-quay.io}
+SAFEIMAGE_USER=${SAFEIMAGE_USER:-libpod}
+SAFEIMAGE_NAME=${SAFEIMAGE_NAME:-testimage}
+SAFEIMAGE_TAG=${SAFEIMAGE_TAG:-20221018}
+SAFEIMAGE="${SAFEIMAGE:-$SAFEIMAGE_REGISTRY/$SAFEIMAGE_USER/$SAFEIMAGE_NAME:$SAFEIMAGE_TAG}"
+
# Shortcut for directory containing Containerfiles for bud.bats
BUDFILES=${TEST_SOURCES}/bud
--
2.37.3

1
vendor/github.com/Microsoft/go-winio/.gitattributes generated vendored Normal file
View File

@ -0,0 +1 @@
* text=auto eol=lf

View File

@ -1 +1,10 @@
.vscode/
*.exe
# testing
testdata
# go workspaces
go.work
go.work.sum

144
vendor/github.com/Microsoft/go-winio/.golangci.yml generated vendored Normal file
View File

@ -0,0 +1,144 @@
run:
skip-dirs:
- pkg/etw/sample
linters:
enable:
# style
- containedctx # struct contains a context
- dupl # duplicate code
- errname # erorrs are named correctly
- goconst # strings that should be constants
- godot # comments end in a period
- misspell
- nolintlint # "//nolint" directives are properly explained
- revive # golint replacement
- stylecheck # golint replacement, less configurable than revive
- unconvert # unnecessary conversions
- wastedassign
# bugs, performance, unused, etc ...
- contextcheck # function uses a non-inherited context
- errorlint # errors not wrapped for 1.13
- exhaustive # check exhaustiveness of enum switch statements
- gofmt # files are gofmt'ed
- gosec # security
- nestif # deeply nested ifs
- nilerr # returns nil even with non-nil error
- prealloc # slices that can be pre-allocated
- structcheck # unused struct fields
- unparam # unused function params
issues:
exclude-rules:
# err is very often shadowed in nested scopes
- linters:
- govet
text: '^shadow: declaration of "err" shadows declaration'
# ignore long lines for skip autogen directives
- linters:
- revive
text: "^line-length-limit: "
source: "^//(go:generate|sys) "
# allow unjustified ignores of error checks in defer statements
- linters:
- nolintlint
text: "^directive `//nolint:errcheck` should provide explanation"
source: '^\s*defer '
# allow unjustified ignores of error lints for io.EOF
- linters:
- nolintlint
text: "^directive `//nolint:errorlint` should provide explanation"
source: '[=|!]= io.EOF'
linters-settings:
govet:
enable-all: true
disable:
# struct order is often for Win32 compat
# also, ignore pointer bytes/GC issues for now until performance becomes an issue
- fieldalignment
check-shadowing: true
nolintlint:
allow-leading-space: false
require-explanation: true
require-specific: true
revive:
# revive is more configurable than static check, so likely the preferred alternative to static-check
# (once the perf issue is solved: https://github.com/golangci/golangci-lint/issues/2997)
enable-all-rules:
true
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
rules:
# rules with required arguments
- name: argument-limit
disabled: true
- name: banned-characters
disabled: true
- name: cognitive-complexity
disabled: true
- name: cyclomatic
disabled: true
- name: file-header
disabled: true
- name: function-length
disabled: true
- name: function-result-limit
disabled: true
- name: max-public-structs
disabled: true
# geneally annoying rules
- name: add-constant # complains about any and all strings and integers
disabled: true
- name: confusing-naming # we frequently use "Foo()" and "foo()" together
disabled: true
- name: flag-parameter # excessive, and a common idiom we use
disabled: true
# general config
- name: line-length-limit
arguments:
- 140
- name: var-naming
arguments:
- []
- - CID
- CRI
- CTRD
- DACL
- DLL
- DOS
- ETW
- FSCTL
- GCS
- GMSA
- HCS
- HV
- IO
- LCOW
- LDAP
- LPAC
- LTSC
- MMIO
- NT
- OCI
- PMEM
- PWSH
- RX
- SACl
- SID
- SMB
- TX
- VHD
- VHDX
- VMID
- VPCI
- WCOW
- WIM
stylecheck:
checks:
- "all"
- "-ST1003" # use revive's var naming

View File

@ -13,16 +13,60 @@ Please see the LICENSE file for licensing information.
## Contributing
This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA)
declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com.
This project welcomes contributions and suggestions.
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that
you have the right to, and actually do, grant us the rights to use your contribution.
For details, visit [Microsoft CLA](https://cla.microsoft.com).
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR
appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
When you submit a pull request, a CLA-bot will automatically determine whether you need to
provide a CLA and decorate the PR appropriately (e.g., label, comment).
Simply follow the instructions provided by the bot.
You will only need to do this once across all repos using our CLA.
We also require that contributors sign their commits using git commit -s or git commit --signoff to certify they either authored the work themselves
or otherwise have permission to use it in this project. Please see https://developercertificate.org/ for more info, as well as to make sure that you can
attest to the rules listed. Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
Additionally, the pull request pipeline requires the following steps to be performed before
mergining.
### Code Sign-Off
We require that contributors sign their commits using [`git commit --signoff`][git-commit-s]
to certify they either authored the work themselves or otherwise have permission to use it in this project.
A range of commits can be signed off using [`git rebase --signoff`][git-rebase-s].
Please see [the developer certificate](https://developercertificate.org) for more info,
as well as to make sure that you can attest to the rules listed.
Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
### Linting
Code must pass a linting stage, which uses [`golangci-lint`][lint].
The linting settings are stored in [`.golangci.yaml`](./.golangci.yaml), and can be run
automatically with VSCode by adding the following to your workspace or folder settings:
```json
"go.lintTool": "golangci-lint",
"go.lintOnSave": "package",
```
Additional editor [integrations options are also available][lint-ide].
Alternatively, `golangci-lint` can be [installed locally][lint-install] and run from the repo root:
```shell
# use . or specify a path to only lint a package
# to show all lint errors, use flags "--max-issues-per-linter=0 --max-same-issues=0"
> golangci-lint run ./...
```
### Go Generate
The pipeline checks that auto-generated code, via `go generate`, are up to date.
This can be done for the entire repo:
```shell
> go generate ./...
```
## Code of Conduct
@ -30,8 +74,16 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
## Special Thanks
Thanks to natefinch for the inspiration for this library. See https://github.com/natefinch/npipe
for another named pipe implementation.
Thanks to [natefinch][natefinch] for the inspiration for this library.
See [npipe](https://github.com/natefinch/npipe) for another named pipe implementation.
[lint]: https://golangci-lint.run/
[lint-ide]: https://golangci-lint.run/usage/integrations/#editor-integration
[lint-install]: https://golangci-lint.run/usage/install/#local-installation
[git-commit-s]: https://git-scm.com/docs/git-commit#Documentation/git-commit.txt--s
[git-rebase-s]: https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---signoff
[natefinch]: https://github.com/natefinch

41
vendor/github.com/Microsoft/go-winio/SECURITY.md generated vendored Normal file
View File

@ -0,0 +1,41 @@
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
## Security
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
## Reporting Security Issues
**Please do not report security vulnerabilities through public GitHub issues.**
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help us triage your report more quickly.
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
## Preferred Languages
We prefer all communications to be in English.
## Policy
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
<!-- END MICROSOFT SECURITY.MD BLOCK -->

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -7,11 +8,12 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"syscall"
"unicode/utf16"
"golang.org/x/sys/windows"
)
//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
@ -24,7 +26,7 @@ const (
BackupAlternateData
BackupLink
BackupPropertyData
BackupObjectId
BackupObjectId //revive:disable-line:var-naming ID, not Id
BackupReparseData
BackupSparseBlock
BackupTxfsData
@ -34,14 +36,16 @@ const (
StreamSparseAttributes = uint32(8)
)
//nolint:revive // var-naming: ALL_CAPS
const (
WRITE_DAC = 0x40000
WRITE_OWNER = 0x80000
ACCESS_SYSTEM_SECURITY = 0x1000000
WRITE_DAC = windows.WRITE_DAC
WRITE_OWNER = windows.WRITE_OWNER
ACCESS_SYSTEM_SECURITY = windows.ACCESS_SYSTEM_SECURITY
)
// BackupHeader represents a backup stream of a file.
type BackupHeader struct {
//revive:disable-next-line:var-naming ID, not Id
Id uint32 // The backup stream ID
Attributes uint32 // Stream attributes
Size int64 // The size of the stream in bytes
@ -49,8 +53,8 @@ type BackupHeader struct {
Offset int64 // The offset of the stream in the file (for BackupSparseBlock only).
}
type win32StreamId struct {
StreamId uint32
type win32StreamID struct {
StreamID uint32
Attributes uint32
Size uint64
NameSize uint32
@ -71,7 +75,7 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
// it was not completely read.
func (r *BackupStreamReader) Next() (*BackupHeader, error) {
if r.bytesLeft > 0 {
if r.bytesLeft > 0 { //nolint:nestif // todo: flatten this
if s, ok := r.r.(io.Seeker); ok {
// Make sure Seek on io.SeekCurrent sometimes succeeds
// before trying the actual seek.
@ -82,16 +86,16 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
r.bytesLeft = 0
}
}
if _, err := io.Copy(ioutil.Discard, r); err != nil {
if _, err := io.Copy(io.Discard, r); err != nil {
return nil, err
}
}
var wsi win32StreamId
var wsi win32StreamID
if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
return nil, err
}
hdr := &BackupHeader{
Id: wsi.StreamId,
Id: wsi.StreamID,
Attributes: wsi.Attributes,
Size: int64(wsi.Size),
}
@ -102,7 +106,7 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
}
hdr.Name = syscall.UTF16ToString(name)
}
if wsi.StreamId == BackupSparseBlock {
if wsi.StreamID == BackupSparseBlock {
if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
return nil, err
}
@ -147,8 +151,8 @@ func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
return fmt.Errorf("missing %d bytes", w.bytesLeft)
}
name := utf16.Encode([]rune(hdr.Name))
wsi := win32StreamId{
StreamId: hdr.Id,
wsi := win32StreamID{
StreamID: hdr.Id,
Attributes: hdr.Attributes,
Size: uint64(hdr.Size),
NameSize: uint32(len(name) * 2),
@ -203,7 +207,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
var bytesRead uint32
err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
if err != nil {
return 0, &os.PathError{"BackupRead", r.f.Name(), err}
return 0, &os.PathError{Op: "BackupRead", Path: r.f.Name(), Err: err}
}
runtime.KeepAlive(r.f)
if bytesRead == 0 {
@ -216,7 +220,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
// the underlying file.
func (r *BackupFileReader) Close() error {
if r.ctx != 0 {
backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
_ = backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
runtime.KeepAlive(r.f)
r.ctx = 0
}
@ -242,7 +246,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
var bytesWritten uint32
err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
if err != nil {
return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
return 0, &os.PathError{Op: "BackupWrite", Path: w.f.Name(), Err: err}
}
runtime.KeepAlive(w.f)
if int(bytesWritten) != len(b) {
@ -255,7 +259,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
// close the underlying file.
func (w *BackupFileWriter) Close() error {
if w.ctx != 0 {
backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
_ = backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
runtime.KeepAlive(w.f)
w.ctx = 0
}
@ -271,7 +275,13 @@ func OpenForBackup(path string, access uint32, share uint32, createmode uint32)
if err != nil {
return nil, err
}
h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
h, err := syscall.CreateFile(&winPath[0],
access,
share,
nil,
createmode,
syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT,
0)
if err != nil {
err = &os.PathError{Op: "open", Path: path, Err: err}
return nil, err

View File

@ -1,4 +1,3 @@
// +build !windows
// This file only exists to allow go get on non-Windows platforms.
package backuptar

View File

@ -1,3 +1,5 @@
//go:build windows
package backuptar
import (

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package backuptar
@ -7,7 +8,6 @@ import (
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"strconv"
"strings"
@ -18,17 +18,18 @@ import (
"golang.org/x/sys/windows"
)
//nolint:deadcode,varcheck // keep unused constants for potential future use
const (
c_ISUID = 04000 // Set uid
c_ISGID = 02000 // Set gid
c_ISVTX = 01000 // Save text (sticky bit)
c_ISDIR = 040000 // Directory
c_ISFIFO = 010000 // FIFO
c_ISREG = 0100000 // Regular file
c_ISLNK = 0120000 // Symbolic link
c_ISBLK = 060000 // Block special file
c_ISCHR = 020000 // Character special file
c_ISSOCK = 0140000 // Socket
cISUID = 0004000 // Set uid
cISGID = 0002000 // Set gid
cISVTX = 0001000 // Save text (sticky bit)
cISDIR = 0040000 // Directory
cISFIFO = 0010000 // FIFO
cISREG = 0100000 // Regular file
cISLNK = 0120000 // Symbolic link
cISBLK = 0060000 // Block special file
cISCHR = 0020000 // Character special file
cISSOCK = 0140000 // Socket
)
const (
@ -44,7 +45,7 @@ const (
// zeroReader is an io.Reader that always returns 0s.
type zeroReader struct{}
func (zr zeroReader) Read(b []byte) (int, error) {
func (zeroReader) Read(b []byte) (int, error) {
for i := range b {
b[i] = 0
}
@ -55,7 +56,7 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
curOffset := int64(0)
for {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
err = io.ErrUnexpectedEOF
}
if err != nil {
@ -71,8 +72,8 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
}
// archive/tar does not support writing sparse files
// so just write zeroes to catch up to the current offset.
if _, err := io.CopyN(t, zeroReader{}, bhdr.Offset-curOffset); err != nil {
return fmt.Errorf("seek to offset %d: %s", bhdr.Offset, err)
if _, err = io.CopyN(t, zeroReader{}, bhdr.Offset-curOffset); err != nil {
return fmt.Errorf("seek to offset %d: %w", bhdr.Offset, err)
}
if bhdr.Size == 0 {
// A sparse block with size = 0 is used to mark the end of the sparse blocks.
@ -106,7 +107,7 @@ func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *ta
hdr.PAXRecords[hdrCreationTime] = formatPAXTime(time.Unix(0, fileInfo.CreationTime.Nanoseconds()))
if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
hdr.Mode |= c_ISDIR
hdr.Mode |= cISDIR
hdr.Size = 0
hdr.Typeflag = tar.TypeDir
}
@ -116,32 +117,29 @@ func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *ta
// SecurityDescriptorFromTarHeader reads the SDDL associated with the header of the current file
// from the tar header and returns the security descriptor into a byte slice.
func SecurityDescriptorFromTarHeader(hdr *tar.Header) ([]byte, error) {
// Maintaining old SDDL-based behavior for backward
// compatibility. All new tar headers written by this library
// will have raw binary for the security descriptor.
var sd []byte
var err error
if sddl, ok := hdr.PAXRecords[hdrSecurityDescriptor]; ok {
sd, err = winio.SddlToSecurityDescriptor(sddl)
if err != nil {
return nil, err
}
}
if sdraw, ok := hdr.PAXRecords[hdrRawSecurityDescriptor]; ok {
sd, err = base64.StdEncoding.DecodeString(sdraw)
sd, err := base64.StdEncoding.DecodeString(sdraw)
if err != nil {
// Not returning sd as-is in the error-case, as base64.DecodeString
// may return partially decoded data (not nil or empty slice) in case
// of a failure: https://github.com/golang/go/blob/go1.17.7/src/encoding/base64/base64.go#L382-L387
return nil, err
}
return sd, nil
}
return sd, nil
// Maintaining old SDDL-based behavior for backward compatibility. All new
// tar headers written by this library will have raw binary for the security
// descriptor.
if sddl, ok := hdr.PAXRecords[hdrSecurityDescriptor]; ok {
return winio.SddlToSecurityDescriptor(sddl)
}
return nil, nil
}
// ExtendedAttributesFromTarHeader reads the EAs associated with the header of the
// current file from the tar header and returns it as a byte slice.
func ExtendedAttributesFromTarHeader(hdr *tar.Header) ([]byte, error) {
var eas []winio.ExtendedAttribute
var eadata []byte
var err error
var eas []winio.ExtendedAttribute //nolint:prealloc // len(eas) <= len(hdr.PAXRecords); prealloc is wasteful
for k, v := range hdr.PAXRecords {
if !strings.HasPrefix(k, hdrEaPrefix) {
continue
@ -155,13 +153,15 @@ func ExtendedAttributesFromTarHeader(hdr *tar.Header) ([]byte, error) {
Value: data,
})
}
var eaData []byte
var err error
if len(eas) != 0 {
eadata, err = winio.EncodeExtendedAttributes(eas)
eaData, err = winio.EncodeExtendedAttributes(eas)
if err != nil {
return nil, err
}
}
return eadata, nil
return eaData, nil
}
// EncodeReparsePointFromTarHeader reads the ReparsePoint structure from the tar header
@ -182,11 +182,9 @@ func EncodeReparsePointFromTarHeader(hdr *tar.Header) []byte {
//
// The additional Win32 metadata is:
//
// MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
//
// MSWINDOWS.rawsd: The Win32 security descriptor, in raw binary format
//
// MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
// - MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
// - MSWINDOWS.rawsd: The Win32 security descriptor, in raw binary format
// - MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
name = filepath.ToSlash(name)
hdr := BasicInfoHeader(name, size, fileInfo)
@ -209,7 +207,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
var dataHdr *winio.BackupHeader
for dataHdr == nil {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
break
}
if err != nil {
@ -217,21 +215,21 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
}
switch bhdr.Id {
case winio.BackupData:
hdr.Mode |= c_ISREG
hdr.Mode |= cISREG
if !readTwice {
dataHdr = bhdr
}
case winio.BackupSecurity:
sd, err := ioutil.ReadAll(br)
sd, err := io.ReadAll(br)
if err != nil {
return err
}
hdr.PAXRecords[hdrRawSecurityDescriptor] = base64.StdEncoding.EncodeToString(sd)
case winio.BackupReparseData:
hdr.Mode |= c_ISLNK
hdr.Mode |= cISLNK
hdr.Typeflag = tar.TypeSymlink
reparseBuffer, err := ioutil.ReadAll(br)
reparseBuffer, _ := io.ReadAll(br)
rp, err := winio.DecodeReparsePoint(reparseBuffer)
if err != nil {
return err
@ -242,7 +240,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
hdr.Linkname = rp.Target
case winio.BackupEaData:
eab, err := ioutil.ReadAll(br)
eab, err := io.ReadAll(br)
if err != nil {
return err
}
@ -276,7 +274,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
}
for dataHdr == nil {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
break
}
if err != nil {
@ -311,7 +309,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
// range of the file containing the range contents. Finally there is a sparse block stream with
// size = 0 and offset = <file size>.
if dataHdr != nil {
if dataHdr != nil { //nolint:nestif // todo: reduce nesting complexity
// A data stream was found. Copy the data.
// We assume that we will either have a data stream size > 0 XOR have sparse block streams.
if dataHdr.Size > 0 || (dataHdr.Attributes&winio.StreamSparseAttributes) == 0 {
@ -319,13 +317,13 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
return fmt.Errorf("%s: mismatch between file size %d and header size %d", name, size, dataHdr.Size)
}
if _, err = io.Copy(t, br); err != nil {
return fmt.Errorf("%s: copying contents from data stream: %s", name, err)
return fmt.Errorf("%s: copying contents from data stream: %w", name, err)
}
} else if size > 0 {
// As of a recent OS change, BackupRead now returns a data stream for empty sparse files.
// These files have no sparse block streams, so skip the copySparse call if file size = 0.
if err = copySparse(t, br); err != nil {
return fmt.Errorf("%s: copying contents from sparse block stream: %s", name, err)
return fmt.Errorf("%s: copying contents from sparse block stream: %w", name, err)
}
}
}
@ -335,7 +333,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
// been written. In practice, this means that we don't get EA or TXF metadata.
for {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
break
}
if err != nil {
@ -343,35 +341,30 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
}
switch bhdr.Id {
case winio.BackupAlternateData:
altName := bhdr.Name
if strings.HasSuffix(altName, ":$DATA") {
altName = altName[:len(altName)-len(":$DATA")]
}
if (bhdr.Attributes & winio.StreamSparseAttributes) == 0 {
hdr = &tar.Header{
Format: hdr.Format,
Name: name + altName,
Mode: hdr.Mode,
Typeflag: tar.TypeReg,
Size: bhdr.Size,
ModTime: hdr.ModTime,
AccessTime: hdr.AccessTime,
ChangeTime: hdr.ChangeTime,
}
err = t.WriteHeader(hdr)
if err != nil {
return err
}
_, err = io.Copy(t, br)
if err != nil {
return err
}
} else {
if (bhdr.Attributes & winio.StreamSparseAttributes) != 0 {
// Unsupported for now, since the size of the alternate stream is not present
// in the backup stream until after the data has been read.
return fmt.Errorf("%s: tar of sparse alternate data streams is unsupported", name)
}
altName := strings.TrimSuffix(bhdr.Name, ":$DATA")
hdr = &tar.Header{
Format: hdr.Format,
Name: name + altName,
Mode: hdr.Mode,
Typeflag: tar.TypeReg,
Size: bhdr.Size,
ModTime: hdr.ModTime,
AccessTime: hdr.AccessTime,
ChangeTime: hdr.ChangeTime,
}
err = t.WriteHeader(hdr)
if err != nil {
return err
}
_, err = io.Copy(t, br)
if err != nil {
return err
}
case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData:
// ignore these streams
default:
@ -413,7 +406,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
}
fileInfo.CreationTime = windows.NsecToFiletime(creationTime.UnixNano())
}
return
return name, size, fileInfo, err
}
// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
@ -474,7 +467,6 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
if err != nil {
return nil, err
}
}
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {

22
vendor/github.com/Microsoft/go-winio/doc.go generated vendored Normal file
View File

@ -0,0 +1,22 @@
// This package provides utilities for efficiently performing Win32 IO operations in Go.
// Currently, this package is provides support for genreal IO and management of
// - named pipes
// - files
// - [Hyper-V sockets]
//
// This code is similar to Go's [net] package, and uses IO completion ports to avoid
// blocking IO on system threads, allowing Go to reuse the thread to schedule other goroutines.
//
// This limits support to Windows Vista and newer operating systems.
//
// Additionally, this package provides support for:
// - creating and managing GUIDs
// - writing to [ETW]
// - opening and manageing VHDs
// - parsing [Windows Image files]
// - auto-generating Win32 API code
//
// [Hyper-V sockets]: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service
// [ETW]: https://docs.microsoft.com/en-us/windows-hardware/drivers/devtest/event-tracing-for-windows--etw-
// [Windows Image files]: https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/work-with-windows-images
package winio

View File

@ -33,7 +33,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
if err != nil {
err = errInvalidEaBuffer
return
return ea, nb, err
}
nameOffset := fileFullEaInformationSize
@ -43,7 +43,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
nextOffset := int(info.NextEntryOffset)
if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
err = errInvalidEaBuffer
return
return ea, nb, err
}
ea.Name = string(b[nameOffset : nameOffset+nameLen])
@ -52,7 +52,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
if info.NextEntryOffset != 0 {
nb = b[info.NextEntryOffset:]
}
return
return ea, nb, err
}
// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
@ -67,7 +67,7 @@ func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
eas = append(eas, ea)
b = nb
}
return
return eas, err
}
func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {

View File

@ -11,6 +11,8 @@ import (
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/windows"
)
//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
@ -24,6 +26,8 @@ type atomicBool int32
func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) }
func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) }
//revive:disable-next-line:predeclared Keep "new" to maintain consistency with "atomic" pkg
func (b *atomicBool) swap(new bool) bool {
var newInt int32
if new {
@ -32,11 +36,6 @@ func (b *atomicBool) swap(new bool) bool {
return atomic.SwapInt32((*int32)(b), newInt) == 1
}
const (
cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
cFILE_SKIP_SET_EVENT_ON_HANDLE = 2
)
var (
ErrFileClosed = errors.New("file has already been closed")
ErrTimeout = &timeoutError{}
@ -44,28 +43,28 @@ var (
type timeoutError struct{}
func (e *timeoutError) Error() string { return "i/o timeout" }
func (e *timeoutError) Timeout() bool { return true }
func (e *timeoutError) Temporary() bool { return true }
func (*timeoutError) Error() string { return "i/o timeout" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return true }
type timeoutChan chan struct{}
var ioInitOnce sync.Once
var ioCompletionPort syscall.Handle
// ioResult contains the result of an asynchronous IO operation
// ioResult contains the result of an asynchronous IO operation.
type ioResult struct {
bytes uint32
err error
}
// ioOperation represents an outstanding asynchronous Win32 IO
// ioOperation represents an outstanding asynchronous Win32 IO.
type ioOperation struct {
o syscall.Overlapped
ch chan ioResult
}
func initIo() {
func initIO() {
h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
if err != nil {
panic(err)
@ -94,15 +93,15 @@ type deadlineHandler struct {
timedout atomicBool
}
// makeWin32File makes a new win32File from an existing file handle
// makeWin32File makes a new win32File from an existing file handle.
func makeWin32File(h syscall.Handle) (*win32File, error) {
f := &win32File{handle: h}
ioInitOnce.Do(initIo)
ioInitOnce.Do(initIO)
_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
if err != nil {
return nil, err
}
err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
err = setFileCompletionNotificationModes(h, windows.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS|windows.FILE_SKIP_SET_EVENT_ON_HANDLE)
if err != nil {
return nil, err
}
@ -121,14 +120,14 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
return f, nil
}
// closeHandle closes the resources associated with a Win32 handle
// closeHandle closes the resources associated with a Win32 handle.
func (f *win32File) closeHandle() {
f.wgLock.Lock()
// Atomically set that we are closing, releasing the resources only once.
if !f.closing.swap(true) {
f.wgLock.Unlock()
// cancel all IO and wait for it to complete
cancelIoEx(f.handle, nil)
_ = cancelIoEx(f.handle, nil)
f.wg.Wait()
// at this point, no new IO can start
syscall.Close(f.handle)
@ -144,14 +143,14 @@ func (f *win32File) Close() error {
return nil
}
// IsClosed checks if the file has been closed
// IsClosed checks if the file has been closed.
func (f *win32File) IsClosed() bool {
return f.closing.isSet()
}
// prepareIo prepares for a new IO operation.
// prepareIO prepares for a new IO operation.
// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
func (f *win32File) prepareIo() (*ioOperation, error) {
func (f *win32File) prepareIO() (*ioOperation, error) {
f.wgLock.RLock()
if f.closing.isSet() {
f.wgLock.RUnlock()
@ -164,7 +163,7 @@ func (f *win32File) prepareIo() (*ioOperation, error) {
return c, nil
}
// ioCompletionProcessor processes completed async IOs forever
// ioCompletionProcessor processes completed async IOs forever.
func ioCompletionProcessor(h syscall.Handle) {
for {
var bytes uint32
@ -178,15 +177,17 @@ func ioCompletionProcessor(h syscall.Handle) {
}
}
// asyncIo processes the return value from ReadFile or WriteFile, blocking until
// todo: helsaawy - create an asyncIO version that takes a context
// asyncIO processes the return value from ReadFile or WriteFile, blocking until
// the operation has actually completed.
func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING {
func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING { //nolint:errorlint // err is Errno
return int(bytes), err
}
if f.closing.isSet() {
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
}
var timeout timeoutChan
@ -200,7 +201,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
select {
case r = <-c.ch:
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
if f.closing.isSet() {
err = ErrFileClosed
}
@ -210,10 +211,10 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
err = wsaGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
}
case <-timeout:
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
r = <-c.ch
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
err = ErrTimeout
}
}
@ -221,13 +222,14 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
// runtime.KeepAlive is needed, as c is passed via native
// code to ioCompletionProcessor, c must remain alive
// until the channel read is complete.
// todo: (de)allocate *ioOperation via win32 heap functions, instead of needing to KeepAlive?
runtime.KeepAlive(c)
return int(r.bytes), err
}
// Read reads from a file handle.
func (f *win32File) Read(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
@ -239,13 +241,13 @@ func (f *win32File) Read(b []byte) (int, error) {
var bytes uint32
err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
n, err := f.asyncIO(c, &f.readDeadline, bytes, err)
runtime.KeepAlive(b)
// Handle EOF conditions.
if err == nil && n == 0 && len(b) != 0 {
return 0, io.EOF
} else if err == syscall.ERROR_BROKEN_PIPE {
} else if err == syscall.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno
return 0, io.EOF
} else {
return n, err
@ -254,7 +256,7 @@ func (f *win32File) Read(b []byte) (int, error) {
// Write writes to a file handle.
func (f *win32File) Write(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
@ -266,7 +268,7 @@ func (f *win32File) Write(b []byte) (int, error) {
var bytes uint32
err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
n, err := f.asyncIO(c, &f.writeDeadline, bytes, err)
runtime.KeepAlive(b)
return n, err
}

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -14,13 +15,18 @@ import (
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
pad uint32 // padding
_ uint32 // padding
}
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &FileBasicInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -29,7 +35,12 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.SetFileInformationByHandle(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -48,7 +59,10 @@ type FileStandardInfo struct {
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()),
windows.FileStandardInfo,
(*byte)(unsafe.Pointer(si)),
uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -65,7 +79,12 @@ type FileIDInfo struct {
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileIdInfo,
(*byte)(unsafe.Pointer(fileID)),
uint32(unsafe.Sizeof(*fileID)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)

View File

@ -4,6 +4,8 @@
package winio
import (
"context"
"errors"
"fmt"
"io"
"net"
@ -12,16 +14,87 @@ import (
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/socket"
"github.com/Microsoft/go-winio/pkg/guid"
)
//sys bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const afHVSock = 34 // AF_HYPERV
const (
afHvSock = 34 // AF_HYPERV
// Well known Service and VM IDs
//https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards
socketError = ^uintptr(0)
)
// HvsockGUIDWildcard is the wildcard VmId for accepting connections from all partitions.
func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000
return guid.GUID{}
}
// HvsockGUIDBroadcast is the wildcard VmId for broadcasting sends to all partitions.
func HvsockGUIDBroadcast() guid.GUID { //ffffffff-ffff-ffff-ffff-ffffffffffff
return guid.GUID{
Data1: 0xffffffff,
Data2: 0xffff,
Data3: 0xffff,
Data4: [8]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
}
}
// HvsockGUIDLoopback is the Loopback VmId for accepting connections to the same partition as the connector.
func HvsockGUIDLoopback() guid.GUID { // e0e16197-dd56-4a10-9195-5ee7a155a838
return guid.GUID{
Data1: 0xe0e16197,
Data2: 0xdd56,
Data3: 0x4a10,
Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38},
}
}
// HvsockGUIDSiloHost is the address of a silo's host partition:
// - The silo host of a hosted silo is the utility VM.
// - The silo host of a silo on a physical host is the physical host.
func HvsockGUIDSiloHost() guid.GUID { // 36bd0c5c-7276-4223-88ba-7d03b654c568
return guid.GUID{
Data1: 0x36bd0c5c,
Data2: 0x7276,
Data3: 0x4223,
Data4: [8]byte{0x88, 0xba, 0x7d, 0x03, 0xb6, 0x54, 0xc5, 0x68},
}
}
// HvsockGUIDChildren is the wildcard VmId for accepting connections from the connector's child partitions.
func HvsockGUIDChildren() guid.GUID { // 90db8b89-0d35-4f79-8ce9-49ea0ac8b7cd
return guid.GUID{
Data1: 0x90db8b89,
Data2: 0xd35,
Data3: 0x4f79,
Data4: [8]uint8{0x8c, 0xe9, 0x49, 0xea, 0xa, 0xc8, 0xb7, 0xcd},
}
}
// HvsockGUIDParent is the wildcard VmId for accepting connections from the connector's parent partition.
// Listening on this VmId accepts connection from:
// - Inside silos: silo host partition.
// - Inside hosted silo: host of the VM.
// - Inside VM: VM host.
// - Physical host: Not supported.
func HvsockGUIDParent() guid.GUID { // a42e7cda-d03f-480c-9cc2-a4de20abb878
return guid.GUID{
Data1: 0xa42e7cda,
Data2: 0xd03f,
Data3: 0x480c,
Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78},
}
}
// hvsockVsockServiceTemplate is the Service GUID used for the VSOCK protocol.
func hvsockVsockServiceTemplate() guid.GUID { // 00000000-facb-11e6-bd58-64006a7986d3
return guid.GUID{
Data2: 0xfacb,
Data3: 0x11e6,
Data4: [8]uint8{0xbd, 0x58, 0x64, 0x00, 0x6a, 0x79, 0x86, 0xd3},
}
}
// An HvsockAddr is an address for a AF_HYPERV socket.
type HvsockAddr struct {
@ -36,8 +109,10 @@ type rawHvsockAddr struct {
ServiceID guid.GUID
}
var _ socket.RawSockaddr = &rawHvsockAddr{}
// Network returns the address's network name, "hvsock".
func (addr *HvsockAddr) Network() string {
func (*HvsockAddr) Network() string {
return "hvsock"
}
@ -47,14 +122,14 @@ func (addr *HvsockAddr) String() string {
// VsockServiceID returns an hvsock service ID corresponding to the specified AF_VSOCK port.
func VsockServiceID(port uint32) guid.GUID {
g, _ := guid.FromString("00000000-facb-11e6-bd58-64006a7986d3")
g := hvsockVsockServiceTemplate() // make a copy
g.Data1 = port
return g
}
func (addr *HvsockAddr) raw() rawHvsockAddr {
return rawHvsockAddr{
Family: afHvSock,
Family: afHVSock,
VMID: addr.VMID,
ServiceID: addr.ServiceID,
}
@ -65,20 +140,48 @@ func (addr *HvsockAddr) fromRaw(raw *rawHvsockAddr) {
addr.ServiceID = raw.ServiceID
}
// Sockaddr returns a pointer to and the size of this struct.
//
// Implements the [socket.RawSockaddr] interface, and allows use in
// [socket.Bind] and [socket.ConnectEx].
func (r *rawHvsockAddr) Sockaddr() (unsafe.Pointer, int32, error) {
return unsafe.Pointer(r), int32(unsafe.Sizeof(rawHvsockAddr{})), nil
}
// Sockaddr interface allows use with `sockets.Bind()` and `.ConnectEx()`.
func (r *rawHvsockAddr) FromBytes(b []byte) error {
n := int(unsafe.Sizeof(rawHvsockAddr{}))
if len(b) < n {
return fmt.Errorf("got %d, want %d: %w", len(b), n, socket.ErrBufferSize)
}
copy(unsafe.Slice((*byte)(unsafe.Pointer(r)), n), b[:n])
if r.Family != afHVSock {
return fmt.Errorf("got %d, want %d: %w", r.Family, afHVSock, socket.ErrAddrFamily)
}
return nil
}
// HvsockListener is a socket listener for the AF_HYPERV address family.
type HvsockListener struct {
sock *win32File
addr HvsockAddr
}
var _ net.Listener = &HvsockListener{}
// HvsockConn is a connected socket of the AF_HYPERV address family.
type HvsockConn struct {
sock *win32File
local, remote HvsockAddr
}
func newHvSocket() (*win32File, error) {
fd, err := syscall.Socket(afHvSock, syscall.SOCK_STREAM, 1)
var _ net.Conn = &HvsockConn{}
func newHVSocket() (*win32File, error) {
fd, err := syscall.Socket(afHVSock, syscall.SOCK_STREAM, 1)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
@ -94,12 +197,12 @@ func newHvSocket() (*win32File, error) {
// ListenHvsock listens for connections on the specified hvsock address.
func ListenHvsock(addr *HvsockAddr) (_ *HvsockListener, err error) {
l := &HvsockListener{addr: *addr}
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("listen", err)
}
sa := addr.raw()
err = bind(sock.handle, unsafe.Pointer(&sa), int32(unsafe.Sizeof(sa)))
err = socket.Bind(windows.Handle(sock.handle), &sa)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("socket", err))
}
@ -121,7 +224,7 @@ func (l *HvsockListener) Addr() net.Addr {
// Accept waits for the next connection and returns it.
func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("accept", err)
}
@ -130,27 +233,42 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock.Close()
}
}()
c, err := l.sock.prepareIo()
c, err := l.sock.prepareIO()
if err != nil {
return nil, l.opErr("accept", err)
}
defer l.sock.wg.Done()
// AcceptEx, per documentation, requires an extra 16 bytes per address.
//
// https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-acceptex
const addrlen = uint32(16 + unsafe.Sizeof(rawHvsockAddr{}))
var addrbuf [addrlen * 2]byte
var bytes uint32
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0, addrlen, addrlen, &bytes, &c.o)
_, err = l.sock.asyncIo(c, nil, bytes, err)
if err != nil {
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /*rxdatalen*/, addrlen, addrlen, &bytes, &c.o)
if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil {
return nil, l.opErr("accept", os.NewSyscallError("acceptex", err))
}
conn := &HvsockConn{
sock: sock,
}
// The local address returned in the AcceptEx buffer is the same as the Listener socket's
// address. However, the service GUID reported by GetSockName is different from the Listeners
// socket, and is sometimes the same as the local address of the socket that dialed the
// address, with the service GUID.Data1 incremented, but othertimes is different.
// todo: does the local address matter? is the listener's address or the actual address appropriate?
conn.local.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[0])))
conn.remote.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[addrlen])))
// initialize the accepted socket and update its properties with those of the listening socket
if err = windows.Setsockopt(windows.Handle(sock.handle),
windows.SOL_SOCKET, windows.SO_UPDATE_ACCEPT_CONTEXT,
(*byte)(unsafe.Pointer(&l.sock.handle)), int32(unsafe.Sizeof(l.sock.handle))); err != nil {
return nil, conn.opErr("accept", os.NewSyscallError("setsockopt", err))
}
sock = nil
return conn, nil
}
@ -160,43 +278,171 @@ func (l *HvsockListener) Close() error {
return l.sock.Close()
}
/* Need to finish ConnectEx handling
func DialHvsock(ctx context.Context, addr *HvsockAddr) (*HvsockConn, error) {
sock, err := newHvSocket()
// HvsockDialer configures and dials a Hyper-V Socket (ie, [HvsockConn]).
type HvsockDialer struct {
// Deadline is the time the Dial operation must connect before erroring.
Deadline time.Time
// Retries is the number of additional connects to try if the connection times out, is refused,
// or the host is unreachable
Retries uint
// RetryWait is the time to wait after a connection error to retry
RetryWait time.Duration
rt *time.Timer // redial wait timer
}
// Dial the Hyper-V socket at addr.
//
// See [HvsockDialer.Dial] for more information.
func Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
return (&HvsockDialer{}).Dial(ctx, addr)
}
// Dial attempts to connect to the Hyper-V socket at addr, and returns a connection if successful.
// Will attempt (HvsockDialer).Retries if dialing fails, waiting (HvsockDialer).RetryWait between
// retries.
//
// Dialing can be cancelled either by providing (HvsockDialer).Deadline, or cancelling ctx.
func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
op := "dial"
// create the conn early to use opErr()
conn = &HvsockConn{
remote: *addr,
}
if !d.Deadline.IsZero() {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, d.Deadline)
defer cancel()
}
// preemptive timeout/cancellation check
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
sock, err := newHVSocket()
if err != nil {
return nil, err
return nil, conn.opErr(op, err)
}
defer func() {
if sock != nil {
sock.Close()
}
}()
c, err := sock.prepareIo()
sa := addr.raw()
err = socket.Bind(windows.Handle(sock.handle), &sa)
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("bind", err))
}
c, err := sock.prepareIO()
if err != nil {
return nil, conn.opErr(op, err)
}
defer sock.wg.Done()
var bytes uint32
err = windows.ConnectEx(windows.Handle(sock.handle), sa, nil, 0, &bytes, &c.o)
_, err = sock.asyncIo(ctx, c, nil, bytes, err)
for i := uint(0); i <= d.Retries; i++ {
err = socket.ConnectEx(
windows.Handle(sock.handle),
&sa,
nil, // sendBuf
0, // sendDataLen
&bytes,
(*windows.Overlapped)(unsafe.Pointer(&c.o)))
_, err = sock.asyncIO(c, nil, bytes, err)
if i < d.Retries && canRedial(err) {
if err = d.redialWait(ctx); err == nil {
continue
}
}
break
}
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("connectex", err))
}
conn := &HvsockConn{
sock: sock,
remote: *addr,
// update the connection properties, so shutdown can be used
if err = windows.Setsockopt(
windows.Handle(sock.handle),
windows.SOL_SOCKET,
windows.SO_UPDATE_CONNECT_CONTEXT,
nil, // optvalue
0, // optlen
); err != nil {
return nil, conn.opErr(op, os.NewSyscallError("setsockopt", err))
}
// get the local name
var sal rawHvsockAddr
err = socket.GetSockName(windows.Handle(sock.handle), &sal)
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("getsockname", err))
}
conn.local.fromRaw(&sal)
// one last check for timeout, since asyncIO doesn't check the context
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
conn.sock = sock
sock = nil
return conn, nil
}
*/
// redialWait waits before attempting to redial, resetting the timer as appropriate.
func (d *HvsockDialer) redialWait(ctx context.Context) (err error) {
if d.RetryWait == 0 {
return nil
}
if d.rt == nil {
d.rt = time.NewTimer(d.RetryWait)
} else {
// should already be stopped and drained
d.rt.Reset(d.RetryWait)
}
select {
case <-ctx.Done():
case <-d.rt.C:
return nil
}
// stop and drain the timer
if !d.rt.Stop() {
<-d.rt.C
}
return ctx.Err()
}
// assumes error is a plain, unwrapped syscall.Errno provided by direct syscall.
func canRedial(err error) bool {
//nolint:errorlint // guaranteed to be an Errno
switch err {
case windows.WSAECONNREFUSED, windows.WSAENETUNREACH, windows.WSAETIMEDOUT,
windows.ERROR_CONNECTION_REFUSED, windows.ERROR_CONNECTION_UNAVAIL:
return true
default:
return false
}
}
func (conn *HvsockConn) opErr(op string, err error) error {
// translate from "file closed" to "socket closed"
if errors.Is(err, ErrFileClosed) {
err = socket.ErrSocketClosed
}
return &net.OpError{Op: op, Net: "hvsock", Source: &conn.local, Addr: &conn.remote, Err: err}
}
func (conn *HvsockConn) Read(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("read", err)
}
@ -204,10 +450,11 @@ func (conn *HvsockConn) Read(b []byte) (int, error) {
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var flags, bytes uint32
err = syscall.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.readDeadline, bytes, err)
n, err := conn.sock.asyncIO(c, &conn.sock.readDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsarecv", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsarecv", eno)
}
return 0, conn.opErr("read", err)
} else if n == 0 {
@ -230,7 +477,7 @@ func (conn *HvsockConn) Write(b []byte) (int, error) {
}
func (conn *HvsockConn) write(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("write", err)
}
@ -238,10 +485,11 @@ func (conn *HvsockConn) write(b []byte) (int, error) {
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var bytes uint32
err = syscall.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.writeDeadline, bytes, err)
n, err := conn.sock.asyncIO(c, &conn.sock.writeDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsasend", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsasend", eno)
}
return 0, conn.opErr("write", err)
}
@ -257,13 +505,19 @@ func (conn *HvsockConn) IsClosed() bool {
return conn.sock.IsClosed()
}
// shutdown disables sending or receiving on a socket.
func (conn *HvsockConn) shutdown(how int) error {
if conn.IsClosed() {
return ErrFileClosed
return socket.ErrSocketClosed
}
err := syscall.Shutdown(conn.sock.handle, how)
if err != nil {
// If the connection was closed, shutdowns fail with "not connected"
if errors.Is(err, windows.WSAENOTCONN) ||
errors.Is(err, windows.WSAESHUTDOWN) {
err = socket.ErrSocketClosed
}
return os.NewSyscallError("shutdown", err)
}
return nil
@ -273,7 +527,7 @@ func (conn *HvsockConn) shutdown(how int) error {
func (conn *HvsockConn) CloseRead() error {
err := conn.shutdown(syscall.SHUT_RD)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closeread", err)
}
return nil
}
@ -283,7 +537,7 @@ func (conn *HvsockConn) CloseRead() error {
func (conn *HvsockConn) CloseWrite() error {
err := conn.shutdown(syscall.SHUT_WR)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closewrite", err)
}
return nil
}
@ -300,8 +554,13 @@ func (conn *HvsockConn) RemoteAddr() net.Addr {
// SetDeadline implements the net.Conn SetDeadline method.
func (conn *HvsockConn) SetDeadline(t time.Time) error {
conn.SetReadDeadline(t)
conn.SetWriteDeadline(t)
// todo: implement `SetDeadline` for `win32File`
if err := conn.SetReadDeadline(t); err != nil {
return fmt.Errorf("set read deadline: %w", err)
}
if err := conn.SetWriteDeadline(t); err != nil {
return fmt.Errorf("set write deadline: %w", err)
}
return nil
}

View File

@ -0,0 +1,20 @@
package socket
import (
"unsafe"
)
// RawSockaddr allows structs to be used with [Bind] and [ConnectEx]. The
// struct must meet the Win32 sockaddr requirements specified here:
// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2
//
// Specifically, the struct size must be least larger than an int16 (unsigned short)
// for the address family.
type RawSockaddr interface {
// Sockaddr returns a pointer to the RawSockaddr and its struct size, allowing
// for the RawSockaddr's data to be overwritten by syscalls (if necessary).
//
// It is the callers responsibility to validate that the values are valid; invalid
// pointers or size can cause a panic.
Sockaddr() (unsafe.Pointer, int32, error)
}

View File

@ -0,0 +1,179 @@
//go:build windows
package socket
import (
"errors"
"fmt"
"net"
"sync"
"syscall"
"unsafe"
"github.com/Microsoft/go-winio/pkg/guid"
"golang.org/x/sys/windows"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go socket.go
//sys getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getsockname
//sys getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getpeername
//sys bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const socketError = uintptr(^uint32(0))
var (
// todo(helsaawy): create custom error types to store the desired vs actual size and addr family?
ErrBufferSize = errors.New("buffer size")
ErrAddrFamily = errors.New("address family")
ErrInvalidPointer = errors.New("invalid pointer")
ErrSocketClosed = fmt.Errorf("socket closed: %w", net.ErrClosed)
)
// todo(helsaawy): replace these with generics, ie: GetSockName[S RawSockaddr](s windows.Handle) (S, error)
// GetSockName writes the local address of socket s to the [RawSockaddr] rsa.
// If rsa is not large enough, the [windows.WSAEFAULT] is returned.
func GetSockName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
// although getsockname returns WSAEFAULT if the buffer is too small, it does not set
// &l to the correct size, so--apart from doubling the buffer repeatedly--there is no remedy
return getsockname(s, ptr, &l)
}
// GetPeerName returns the remote address the socket is connected to.
//
// See [GetSockName] for more information.
func GetPeerName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return getpeername(s, ptr, &l)
}
func Bind(s windows.Handle, rsa RawSockaddr) (err error) {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return bind(s, ptr, l)
}
// "golang.org/x/sys/windows".ConnectEx and .Bind only accept internal implementations of the
// their sockaddr interface, so they cannot be used with HvsockAddr
// Replicate functionality here from
// https://cs.opensource.google/go/x/sys/+/master:windows/syscall_windows.go
// The function pointers to `AcceptEx`, `ConnectEx` and `GetAcceptExSockaddrs` must be loaded at
// runtime via a WSAIoctl call:
// https://docs.microsoft.com/en-us/windows/win32/api/Mswsock/nc-mswsock-lpfn_connectex#remarks
type runtimeFunc struct {
id guid.GUID
once sync.Once
addr uintptr
err error
}
func (f *runtimeFunc) Load() error {
f.once.Do(func() {
var s windows.Handle
s, f.err = windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP)
if f.err != nil {
return
}
defer windows.CloseHandle(s) //nolint:errcheck
var n uint32
f.err = windows.WSAIoctl(s,
windows.SIO_GET_EXTENSION_FUNCTION_POINTER,
(*byte)(unsafe.Pointer(&f.id)),
uint32(unsafe.Sizeof(f.id)),
(*byte)(unsafe.Pointer(&f.addr)),
uint32(unsafe.Sizeof(f.addr)),
&n,
nil, //overlapped
0, //completionRoutine
)
})
return f.err
}
var (
// todo: add `AcceptEx` and `GetAcceptExSockaddrs`
WSAID_CONNECTEX = guid.GUID{ //revive:disable-line:var-naming ALL_CAPS
Data1: 0x25a207b9,
Data2: 0xddf3,
Data3: 0x4660,
Data4: [8]byte{0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e},
}
connectExFunc = runtimeFunc{id: WSAID_CONNECTEX}
)
func ConnectEx(
fd windows.Handle,
rsa RawSockaddr,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) error {
if err := connectExFunc.Load(); err != nil {
return fmt.Errorf("failed to load ConnectEx function pointer: %w", err)
}
ptr, n, err := rsa.Sockaddr()
if err != nil {
return err
}
return connectEx(fd, ptr, n, sendBuf, sendDataLen, bytesSent, overlapped)
}
// BOOL LpfnConnectex(
// [in] SOCKET s,
// [in] const sockaddr *name,
// [in] int namelen,
// [in, optional] PVOID lpSendBuffer,
// [in] DWORD dwSendDataLength,
// [out] LPDWORD lpdwBytesSent,
// [in] LPOVERLAPPED lpOverlapped
// )
func connectEx(
s windows.Handle,
name unsafe.Pointer,
namelen int32,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) (err error) {
// todo: after upgrading to 1.18, switch from syscall.Syscall9 to syscall.SyscallN
r1, _, e1 := syscall.Syscall9(connectExFunc.addr,
7,
uintptr(s),
uintptr(name),
uintptr(namelen),
uintptr(unsafe.Pointer(sendBuf)),
uintptr(sendDataLen),
uintptr(unsafe.Pointer(bytesSent)),
uintptr(unsafe.Pointer(overlapped)),
0,
0)
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return err
}

View File

@ -0,0 +1,72 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package socket
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
// TODO: add more here, after collecting data on the common
// error values see on Windows. (perhaps when running
// all.bat?)
return e
}
var (
modws2_32 = windows.NewLazySystemDLL("ws2_32.dll")
procbind = modws2_32.NewProc("bind")
procgetpeername = modws2_32.NewProc("getpeername")
procgetsockname = modws2_32.NewProc("getsockname")
)
func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.Syscall(procgetpeername.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.Syscall(procgetsockname.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -13,6 +14,8 @@ import (
"syscall"
"time"
"unsafe"
"golang.org/x/sys/windows"
)
//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
@ -21,10 +24,10 @@ import (
//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntstatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) = ntdll.RtlDefaultNpAcl
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntStatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) = ntdll.RtlDefaultNpAcl
type ioStatusBlock struct {
Status, Information uintptr
@ -51,45 +54,22 @@ type securityDescriptor struct {
Control uint16
Owner uintptr
Group uintptr
Sacl uintptr
Dacl uintptr
Sacl uintptr //revive:disable-line:var-naming SACL, not Sacl
Dacl uintptr //revive:disable-line:var-naming DACL, not Dacl
}
type ntstatus int32
type ntStatus int32
func (status ntstatus) Err() error {
func (status ntStatus) Err() error {
if status >= 0 {
return nil
}
return rtlNtStatusToDosError(status)
}
const (
cERROR_PIPE_BUSY = syscall.Errno(231)
cERROR_NO_DATA = syscall.Errno(232)
cERROR_PIPE_CONNECTED = syscall.Errno(535)
cERROR_SEM_TIMEOUT = syscall.Errno(121)
cSECURITY_SQOS_PRESENT = 0x100000
cSECURITY_ANONYMOUS = 0
cPIPE_TYPE_MESSAGE = 4
cPIPE_READMODE_MESSAGE = 2
cFILE_OPEN = 1
cFILE_CREATE = 2
cFILE_PIPE_MESSAGE_TYPE = 1
cFILE_PIPE_REJECT_REMOTE_CLIENTS = 2
cSE_DACL_PRESENT = 4
)
var (
// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
// This error should match net.errClosing since docker takes a dependency on its text.
ErrPipeListenerClosed = errors.New("use of closed network connection")
ErrPipeListenerClosed = net.ErrClosed
errPipeWriteClosed = errors.New("pipe has been closed for write")
)
@ -116,9 +96,10 @@ func (f *win32Pipe) RemoteAddr() net.Addr {
}
func (f *win32Pipe) SetDeadline(t time.Time) error {
f.SetReadDeadline(t)
f.SetWriteDeadline(t)
return nil
if err := f.SetReadDeadline(t); err != nil {
return err
}
return f.SetWriteDeadline(t)
}
// CloseWrite closes the write side of a message pipe in byte mode.
@ -157,14 +138,14 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return 0, io.EOF
}
n, err := f.win32File.Read(b)
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
// If this was the result of a zero-byte read, then
// it is possible that the read was due to a zero-size
// message. Since we are simulating CloseWrite with a
// zero-byte message, ensure that all future Read() calls
// also return EOF.
f.readEOF = true
} else if err == syscall.ERROR_MORE_DATA {
} else if err == syscall.ERROR_MORE_DATA { //nolint:errorlint // err is Errno
// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
// and the message still has more bytes. Treat this as a success, since
// this package presents all named pipes as byte streams.
@ -173,7 +154,7 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return n, err
}
func (s pipeAddress) Network() string {
func (pipeAddress) Network() string {
return "pipe"
}
@ -184,16 +165,21 @@ func (s pipeAddress) String() string {
// tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
func tryDialPipe(ctx context.Context, path *string, access uint32) (syscall.Handle, error) {
for {
select {
case <-ctx.Done():
return syscall.Handle(0), ctx.Err()
default:
h, err := createFile(*path, access, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
h, err := createFile(*path,
access,
0,
nil,
syscall.OPEN_EXISTING,
windows.FILE_FLAG_OVERLAPPED|windows.SECURITY_SQOS_PRESENT|windows.SECURITY_ANONYMOUS,
0)
if err == nil {
return h, nil
}
if err != cERROR_PIPE_BUSY {
if err != windows.ERROR_PIPE_BUSY { //nolint:errorlint // err is Errno
return h, &os.PathError{Err: err, Op: "open", Path: *path}
}
// Wait 10 msec and try again. This is a rather simplistic
@ -213,9 +199,10 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
} else {
absTimeout = time.Now().Add(2 * time.Second)
}
ctx, _ := context.WithDeadline(context.Background(), absTimeout)
ctx, cancel := context.WithDeadline(context.Background(), absTimeout)
defer cancel()
conn, err := DialPipeContext(ctx, path)
if err == context.DeadlineExceeded {
if errors.Is(err, context.DeadlineExceeded) {
return nil, ErrTimeout
}
return conn, err
@ -251,7 +238,7 @@ func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn,
// If the pipe is in message mode, return a message byte pipe, which
// supports CloseWrite().
if flags&cPIPE_TYPE_MESSAGE != 0 {
if flags&windows.PIPE_TYPE_MESSAGE != 0 {
return &win32MessageBytePipe{
win32Pipe: win32Pipe{win32File: f, path: path},
}, nil
@ -283,7 +270,11 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
oa.Length = unsafe.Sizeof(oa)
var ntPath unicodeString
if err := rtlDosPathNameToNtPathName(&path16[0], &ntPath, 0, 0).Err(); err != nil {
if err := rtlDosPathNameToNtPathName(&path16[0],
&ntPath,
0,
0,
).Err(); err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
defer localFree(ntPath.Buffer)
@ -292,8 +283,8 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
// The security descriptor is only needed for the first pipe.
if first {
if sd != nil {
len := uint32(len(sd))
sdb := localAlloc(0, len)
l := uint32(len(sd))
sdb := localAlloc(0, l)
defer localFree(sdb)
copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd)
oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb))
@ -301,28 +292,28 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
// Construct the default named pipe security descriptor.
var dacl uintptr
if err := rtlDefaultNpAcl(&dacl).Err(); err != nil {
return 0, fmt.Errorf("getting default named pipe ACL: %s", err)
return 0, fmt.Errorf("getting default named pipe ACL: %w", err)
}
defer localFree(dacl)
sdb := &securityDescriptor{
Revision: 1,
Control: cSE_DACL_PRESENT,
Control: windows.SE_DACL_PRESENT,
Dacl: dacl,
}
oa.SecurityDescriptor = sdb
}
}
typ := uint32(cFILE_PIPE_REJECT_REMOTE_CLIENTS)
typ := uint32(windows.FILE_PIPE_REJECT_REMOTE_CLIENTS)
if c.MessageMode {
typ |= cFILE_PIPE_MESSAGE_TYPE
typ |= windows.FILE_PIPE_MESSAGE_TYPE
}
disposition := uint32(cFILE_OPEN)
disposition := uint32(windows.FILE_OPEN)
access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | syscall.SYNCHRONIZE)
if first {
disposition = cFILE_CREATE
disposition = windows.FILE_CREATE
// By not asking for read or write access, the named pipe file system
// will put this pipe into an initially disconnected state, blocking
// client connections until the next call with first == false.
@ -335,7 +326,20 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
h syscall.Handle
iosb ioStatusBlock
)
err = ntCreateNamedPipeFile(&h, access, &oa, &iosb, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, disposition, 0, typ, 0, 0, 0xffffffff, uint32(c.InputBufferSize), uint32(c.OutputBufferSize), &timeout).Err()
err = ntCreateNamedPipeFile(&h,
access,
&oa,
&iosb,
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
disposition,
0,
typ,
0,
0,
0xffffffff,
uint32(c.InputBufferSize),
uint32(c.OutputBufferSize),
&timeout).Err()
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
@ -380,7 +384,7 @@ func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
p.Close()
p = nil
err = <-ch
if err == nil || err == ErrFileClosed {
if err == nil || err == ErrFileClosed { //nolint:errorlint // err is Errno
err = ErrPipeListenerClosed
}
}
@ -402,12 +406,12 @@ func (l *win32PipeListener) listenerRoutine() {
p, err = l.makeConnectedServerPipe()
// If the connection was immediately closed by the client, try
// again.
if err != cERROR_NO_DATA {
if err != windows.ERROR_NO_DATA { //nolint:errorlint // err is Errno
break
}
}
responseCh <- acceptResponse{p, err}
closed = err == ErrPipeListenerClosed
closed = err == ErrPipeListenerClosed //nolint:errorlint // err is Errno
}
}
syscall.Close(l.firstHandle)
@ -469,15 +473,15 @@ func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
}
func connectPipe(p *win32File) error {
c, err := p.prepareIo()
c, err := p.prepareIO()
if err != nil {
return err
}
defer p.wg.Done()
err = connectNamedPipe(p.handle, &c.o)
_, err = p.asyncIo(c, nil, 0, err)
if err != nil && err != cERROR_PIPE_CONNECTED {
_, err = p.asyncIO(c, nil, 0, err)
if err != nil && err != windows.ERROR_PIPE_CONNECTED { //nolint:errorlint // err is Errno
return err
}
return nil

View File

@ -1,5 +1,3 @@
// +build windows
// Package guid provides a GUID type. The backing structure for a GUID is
// identical to that used by the golang.org/x/sys/windows GUID type.
// There are two main binary encodings used for a GUID, the big-endian encoding,
@ -9,24 +7,26 @@ package guid
import (
"crypto/rand"
"crypto/sha1"
"crypto/sha1" //nolint:gosec // not used for secure application
"encoding"
"encoding/binary"
"fmt"
"strconv"
)
//go:generate go run golang.org/x/tools/cmd/stringer -type=Variant -trimprefix=Variant -linecomment
// Variant specifies which GUID variant (or "type") of the GUID. It determines
// how the entirety of the rest of the GUID is interpreted.
type Variant uint8
// The variants specified by RFC 4122.
// The variants specified by RFC 4122 section 4.1.1.
const (
// VariantUnknown specifies a GUID variant which does not conform to one of
// the variant encodings specified in RFC 4122.
VariantUnknown Variant = iota
VariantNCS
VariantRFC4122
VariantRFC4122 // RFC 4122
VariantMicrosoft
VariantFuture
)
@ -36,6 +36,10 @@ const (
// hash of an input string.
type Version uint8
func (v Version) String() string {
return strconv.FormatUint(uint64(v), 10)
}
var _ = (encoding.TextMarshaler)(GUID{})
var _ = (encoding.TextUnmarshaler)(&GUID{})
@ -61,7 +65,7 @@ func NewV4() (GUID, error) {
// big-endian UTF16 stream of bytes. If that is desired, the string can be
// encoded as such before being passed to this function.
func NewV5(namespace GUID, name []byte) (GUID, error) {
b := sha1.New()
b := sha1.New() //nolint:gosec // not used for secure application
namespaceBytes := namespace.ToArray()
b.Write(namespaceBytes[:])
b.Write(name)

View File

@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package guid

View File

@ -1,3 +1,6 @@
//go:build windows
// +build windows
package guid
import "golang.org/x/sys/windows"

View File

@ -0,0 +1,27 @@
// Code generated by "stringer -type=Variant -trimprefix=Variant -linecomment"; DO NOT EDIT.
package guid
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[VariantUnknown-0]
_ = x[VariantNCS-1]
_ = x[VariantRFC4122-2]
_ = x[VariantMicrosoft-3]
_ = x[VariantFuture-4]
}
const _Variant_name = "UnknownNCSRFC 4122MicrosoftFuture"
var _Variant_index = [...]uint8{0, 7, 10, 18, 27, 33}
func (i Variant) String() string {
if i >= Variant(len(_Variant_index)-1) {
return "Variant(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Variant_name[_Variant_index[i]:_Variant_index[i+1]]
}

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package security
@ -20,6 +21,7 @@ type (
trusteeForm uint32
trusteeType uint32
//nolint:structcheck // structcheck thinks fields are unused, but the are used to pass data to OS
explicitAccess struct {
accessPermissions accessMask
accessMode accessMode
@ -27,6 +29,7 @@ type (
trustee trustee
}
//nolint:structcheck,unused // structcheck thinks fields are unused, but the are used to pass data to OS
trustee struct {
multipleTrustee *trustee
multipleTrusteeOperation int32
@ -44,6 +47,7 @@ const (
desiredAccessReadControl desiredAccess = 0x20000
desiredAccessWriteDac desiredAccess = 0x40000
//cspell:disable-next-line
gvmga = "GrantVmGroupAccess:"
inheritModeNoInheritance inheritMode = 0x0
@ -56,9 +60,9 @@ const (
shareModeRead shareMode = 0x1
shareModeWrite shareMode = 0x2
sidVmGroup = "S-1-5-83-0"
sidVMGroup = "S-1-5-83-0"
trusteeFormIsSid trusteeForm = 0
trusteeFormIsSID trusteeForm = 0
trusteeTypeWellKnownGroup trusteeType = 5
)
@ -67,6 +71,8 @@ const (
// include Grant ACE entries for the VM Group SID. This is a golang re-
// implementation of the same function in vmcompute, just not exported in
// RS5. Which kind of sucks. Sucks a lot :/
//
//revive:disable-next-line:var-naming VM, not Vm
func GrantVmGroupAccess(name string) error {
// Stat (to determine if `name` is a directory).
s, err := os.Stat(name)
@ -79,7 +85,7 @@ func GrantVmGroupAccess(name string) error {
if err != nil {
return err // Already wrapped
}
defer syscall.CloseHandle(fd)
defer syscall.CloseHandle(fd) //nolint:errcheck
// Get the current DACL and Security Descriptor. Must defer LocalFree on success.
ot := objectTypeFileObject
@ -89,7 +95,7 @@ func GrantVmGroupAccess(name string) error {
if err := getSecurityInfo(fd, uint32(ot), uint32(si), nil, nil, &origDACL, nil, &sd); err != nil {
return fmt.Errorf("%s GetSecurityInfo %s: %w", gvmga, name, err)
}
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(sd)))
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(sd))) //nolint:errcheck
// Generate a new DACL which is the current DACL with the required ACEs added.
// Must defer LocalFree on success.
@ -97,7 +103,7 @@ func GrantVmGroupAccess(name string) error {
if err != nil {
return err // Already wrapped
}
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(newDACL)))
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(newDACL))) //nolint:errcheck
// And finally use SetSecurityInfo to apply the updated DACL.
if err := setSecurityInfo(fd, uint32(ot), uint32(si), uintptr(0), uintptr(0), newDACL, uintptr(0)); err != nil {
@ -110,16 +116,19 @@ func GrantVmGroupAccess(name string) error {
// createFile is a helper function to call [Nt]CreateFile to get a handle to
// the file or directory.
func createFile(name string, isDir bool) (syscall.Handle, error) {
namep := syscall.StringToUTF16(name)
namep, err := syscall.UTF16FromString(name)
if err != nil {
return syscall.InvalidHandle, fmt.Errorf("could not convernt name to UTF-16: %w", err)
}
da := uint32(desiredAccessReadControl | desiredAccessWriteDac)
sm := uint32(shareModeRead | shareModeWrite)
fa := uint32(syscall.FILE_ATTRIBUTE_NORMAL)
if isDir {
fa = uint32(fa | syscall.FILE_FLAG_BACKUP_SEMANTICS)
fa |= syscall.FILE_FLAG_BACKUP_SEMANTICS
}
fd, err := syscall.CreateFile(&namep[0], da, sm, nil, syscall.OPEN_EXISTING, fa, 0)
if err != nil {
return 0, fmt.Errorf("%s syscall.CreateFile %s: %w", gvmga, name, err)
return syscall.InvalidHandle, fmt.Errorf("%s syscall.CreateFile %s: %w", gvmga, name, err)
}
return fd, nil
}
@ -128,9 +137,9 @@ func createFile(name string, isDir bool) (syscall.Handle, error) {
// The caller is responsible for LocalFree of the returned DACL on success.
func generateDACLWithAcesAdded(name string, isDir bool, origDACL uintptr) (uintptr, error) {
// Generate pointers to the SIDs based on the string SIDs
sid, err := syscall.StringToSid(sidVmGroup)
sid, err := syscall.StringToSid(sidVMGroup)
if err != nil {
return 0, fmt.Errorf("%s syscall.StringToSid %s %s: %w", gvmga, name, sidVmGroup, err)
return 0, fmt.Errorf("%s syscall.StringToSid %s %s: %w", gvmga, name, sidVMGroup, err)
}
inheritance := inheritModeNoInheritance
@ -139,12 +148,12 @@ func generateDACLWithAcesAdded(name string, isDir bool, origDACL uintptr) (uintp
}
eaArray := []explicitAccess{
explicitAccess{
{
accessPermissions: accessMaskDesiredPermission,
accessMode: accessModeGrant,
inheritance: inheritance,
trustee: trustee{
trusteeForm: trusteeFormIsSid,
trusteeForm: trusteeFormIsSID,
trusteeType: trusteeTypeWellKnownGroup,
name: uintptr(unsafe.Pointer(sid)),
},

View File

@ -1,6 +1,6 @@
package security
//go:generate go run mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go syscall_windows.go
//sys getSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, ppsidOwner **uintptr, ppsidGroup **uintptr, ppDacl *uintptr, ppSacl *uintptr, ppSecurityDescriptor *uintptr) (win32err error) = advapi32.GetSecurityInfo
//sys setSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, psidOwner uintptr, psidGroup uintptr, pDacl uintptr, pSacl uintptr) (win32err error) = advapi32.SetSecurityInfo

View File

@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package security

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -24,22 +25,17 @@ import (
//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
const (
SE_PRIVILEGE_ENABLED = 2
//revive:disable-next-line:var-naming ALL_CAPS
SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED
ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
//revive:disable-next-line:var-naming ALL_CAPS
ERROR_NOT_ALL_ASSIGNED syscall.Errno = windows.ERROR_NOT_ALL_ASSIGNED
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeSecurityPrivilege = "SeSecurityPrivilege"
)
const (
securityAnonymous = iota
securityIdentification
securityImpersonation
securityDelegation
)
var (
privNames = make(map[string]uint64)
privNameMutex sync.Mutex
@ -51,11 +47,9 @@ type PrivilegeError struct {
}
func (e *PrivilegeError) Error() string {
s := ""
s := "Could not enable privilege "
if len(e.privileges) > 1 {
s = "Could not enable privileges "
} else {
s = "Could not enable privilege "
}
for i, p := range e.privileges {
if i != 0 {
@ -94,7 +88,7 @@ func RunWithPrivileges(names []string, fn func() error) error {
}
func mapPrivileges(names []string) ([]uint64, error) {
var privileges []uint64
privileges := make([]uint64, 0, len(names))
privNameMutex.Lock()
defer privNameMutex.Unlock()
for _, name := range names {
@ -127,7 +121,7 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
return err
}
p, _ := windows.GetCurrentProcess()
p := windows.CurrentProcess()
var token windows.Token
err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
if err != nil {
@ -140,10 +134,10 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
_ = binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
for _, p := range privileges {
binary.Write(&b, binary.LittleEndian, p)
binary.Write(&b, binary.LittleEndian, action)
_ = binary.Write(&b, binary.LittleEndian, p)
_ = binary.Write(&b, binary.LittleEndian, action)
}
prevState := make([]byte, b.Len())
reqSize := uint32(0)
@ -151,7 +145,7 @@ func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) e
if !success {
return err
}
if err == ERROR_NOT_ALL_ASSIGNED {
if err == ERROR_NOT_ALL_ASSIGNED { //nolint:errorlint // err is Errno
return &PrivilegeError{privileges}
}
return nil
@ -177,7 +171,7 @@ func getPrivilegeName(luid uint64) string {
}
func newThreadToken() (windows.Token, error) {
err := impersonateSelf(securityImpersonation)
err := impersonateSelf(windows.SecurityImpersonation)
if err != nil {
return 0, err
}

View File

@ -1,3 +1,6 @@
//go:build windows
// +build windows
package winio
import (
@ -113,16 +116,16 @@ func EncodeReparsePoint(rp *ReparsePoint) []byte {
}
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, &data)
_ = binary.Write(&b, binary.LittleEndian, &data)
if !rp.IsMountPoint {
flags := uint32(0)
if relative {
flags |= 1
}
binary.Write(&b, binary.LittleEndian, flags)
_ = binary.Write(&b, binary.LittleEndian, flags)
}
binary.Write(&b, binary.LittleEndian, ntTarget16)
binary.Write(&b, binary.LittleEndian, target16)
_ = binary.Write(&b, binary.LittleEndian, ntTarget16)
_ = binary.Write(&b, binary.LittleEndian, target16)
return b.Bytes()
}

View File

@ -1,23 +1,25 @@
//go:build windows
// +build windows
package winio
import (
"errors"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
//sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW
//sys lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountSidW
//sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW
//sys convertStringSidToSid(str *uint16, sid **byte) (err error) = advapi32.ConvertStringSidToSidW
//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW
//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW
//sys localFree(mem uintptr) = LocalFree
//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength
const (
cERROR_NONE_MAPPED = syscall.Errno(1332)
)
type AccountLookupError struct {
Name string
Err error
@ -28,8 +30,10 @@ func (e *AccountLookupError) Error() string {
return "lookup account: empty account name specified"
}
var s string
switch e.Err {
case cERROR_NONE_MAPPED:
switch {
case errors.Is(e.Err, windows.ERROR_INVALID_SID):
s = "the security ID structure is invalid"
case errors.Is(e.Err, windows.ERROR_NONE_MAPPED):
s = "not found"
default:
s = e.Err.Error()
@ -37,6 +41,8 @@ func (e *AccountLookupError) Error() string {
return "lookup account " + e.Name + ": " + s
}
func (e *AccountLookupError) Unwrap() error { return e.Err }
type SddlConversionError struct {
Sddl string
Err error
@ -46,15 +52,19 @@ func (e *SddlConversionError) Error() string {
return "convert " + e.Sddl + ": " + e.Err.Error()
}
func (e *SddlConversionError) Unwrap() error { return e.Err }
// LookupSidByName looks up the SID of an account by name
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupSidByName(name string) (sid string, err error) {
if name == "" {
return "", &AccountLookupError{name, cERROR_NONE_MAPPED}
return "", &AccountLookupError{name, windows.ERROR_NONE_MAPPED}
}
var sidSize, sidNameUse, refDomainSize uint32
err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER {
if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{name, err}
}
sidBuffer := make([]byte, sidSize)
@ -73,6 +83,42 @@ func LookupSidByName(name string) (sid string, err error) {
return sid, nil
}
// LookupNameBySid looks up the name of an account by SID
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupNameBySid(sid string) (name string, err error) {
if sid == "" {
return "", &AccountLookupError{sid, windows.ERROR_NONE_MAPPED}
}
sidBuffer, err := windows.UTF16PtrFromString(sid)
if err != nil {
return "", &AccountLookupError{sid, err}
}
var sidPtr *byte
if err = convertStringSidToSid(sidBuffer, &sidPtr); err != nil {
return "", &AccountLookupError{sid, err}
}
defer localFree(uintptr(unsafe.Pointer(sidPtr)))
var nameSize, refDomainSize, sidNameUse uint32
err = lookupAccountSid(nil, sidPtr, nil, &nameSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{sid, err}
}
nameBuffer := make([]uint16, nameSize)
refDomainBuffer := make([]uint16, refDomainSize)
err = lookupAccountSid(nil, sidPtr, &nameBuffer[0], &nameSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
if err != nil {
return "", &AccountLookupError{sid, err}
}
name = windows.UTF16ToString(nameBuffer)
return name, nil
}
func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
var sdBuffer uintptr
err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil)
@ -87,7 +133,7 @@ func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
func SecurityDescriptorToSddl(sd []byte) (string, error) {
var sddl *uint16
// The returned string length seems to including an aribtrary number of terminating NULs.
// The returned string length seems to include an arbitrary number of terminating NULs.
// Don't use it.
err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil)
if err != nil {

View File

@ -1,3 +1,5 @@
//go:build windows
package winio
//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go ./*.go

5
vendor/github.com/Microsoft/go-winio/tools.go generated vendored Normal file
View File

@ -0,0 +1,5 @@
//go:build tools
package winio
import _ "golang.org/x/tools/cmd/stringer"

View File

@ -11,7 +11,7 @@ import (
"golang.org/x/sys/windows"
)
//go:generate go run mksyscall_windows.go -output zvhd_windows.go vhd.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zvhd_windows.go vhd.go
//sys createVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (win32err error) = virtdisk.CreateVirtualDisk
//sys openVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (win32err error) = virtdisk.OpenVirtualDisk
@ -62,8 +62,8 @@ type OpenVirtualDiskParameters struct {
Version2 OpenVersion2
}
// The higher level `OpenVersion2` struct uses bools to refer to `GetInfoOnly` and `ReadOnly` for ease of use. However,
// the internal windows structure uses `BOOLS` aka int32s for these types. `openVersion2` is used for translating
// The higher level `OpenVersion2` struct uses `bool`s to refer to `GetInfoOnly` and `ReadOnly` for ease of use. However,
// the internal windows structure uses `BOOL`s aka int32s for these types. `openVersion2` is used for translating
// `OpenVersion2` fields to the correct windows internal field types on the `Open____` methods.
type openVersion2 struct {
getInfoOnly int32
@ -87,9 +87,10 @@ type AttachVirtualDiskParameters struct {
}
const (
//revive:disable-next-line:var-naming ALL_CAPS
VIRTUAL_STORAGE_TYPE_DEVICE_VHDX = 0x3
// Access Mask for opening a VHD
// Access Mask for opening a VHD.
VirtualDiskAccessNone VirtualDiskAccessMask = 0x00000000
VirtualDiskAccessAttachRO VirtualDiskAccessMask = 0x00010000
VirtualDiskAccessAttachRW VirtualDiskAccessMask = 0x00020000
@ -101,7 +102,7 @@ const (
VirtualDiskAccessAll VirtualDiskAccessMask = 0x003f0000
VirtualDiskAccessWritable VirtualDiskAccessMask = 0x00320000
// Flags for creating a VHD
// Flags for creating a VHD.
CreateVirtualDiskFlagNone CreateVirtualDiskFlag = 0x0
CreateVirtualDiskFlagFullPhysicalAllocation CreateVirtualDiskFlag = 0x1
CreateVirtualDiskFlagPreventWritesToSourceDisk CreateVirtualDiskFlag = 0x2
@ -109,12 +110,12 @@ const (
CreateVirtualDiskFlagCreateBackingStorage CreateVirtualDiskFlag = 0x8
CreateVirtualDiskFlagUseChangeTrackingSourceLimit CreateVirtualDiskFlag = 0x10
CreateVirtualDiskFlagPreserveParentChangeTrackingState CreateVirtualDiskFlag = 0x20
CreateVirtualDiskFlagVhdSetUseOriginalBackingStorage CreateVirtualDiskFlag = 0x40
CreateVirtualDiskFlagVhdSetUseOriginalBackingStorage CreateVirtualDiskFlag = 0x40 //revive:disable-line:var-naming VHD, not Vhd
CreateVirtualDiskFlagSparseFile CreateVirtualDiskFlag = 0x80
CreateVirtualDiskFlagPmemCompatible CreateVirtualDiskFlag = 0x100
CreateVirtualDiskFlagPmemCompatible CreateVirtualDiskFlag = 0x100 //revive:disable-line:var-naming PMEM, not Pmem
CreateVirtualDiskFlagSupportCompressedVolumes CreateVirtualDiskFlag = 0x200
// Flags for opening a VHD
// Flags for opening a VHD.
OpenVirtualDiskFlagNone VirtualDiskFlag = 0x00000000
OpenVirtualDiskFlagNoParents VirtualDiskFlag = 0x00000001
OpenVirtualDiskFlagBlankFile VirtualDiskFlag = 0x00000002
@ -127,7 +128,7 @@ const (
OpenVirtualDiskFlagNoWriteHardening VirtualDiskFlag = 0x00000100
OpenVirtualDiskFlagSupportCompressedVolumes VirtualDiskFlag = 0x00000200
// Flags for attaching a VHD
// Flags for attaching a VHD.
AttachVirtualDiskFlagNone AttachVirtualDiskFlag = 0x00000000
AttachVirtualDiskFlagReadOnly AttachVirtualDiskFlag = 0x00000001
AttachVirtualDiskFlagNoDriveLetter AttachVirtualDiskFlag = 0x00000002
@ -140,12 +141,14 @@ const (
AttachVirtualDiskFlagSinglePartition AttachVirtualDiskFlag = 0x00000100
AttachVirtualDiskFlagRegisterVolume AttachVirtualDiskFlag = 0x00000200
// Flags for detaching a VHD
// Flags for detaching a VHD.
DetachVirtualDiskFlagNone DetachVirtualDiskFlag = 0x0
)
// CreateVhdx is a helper function to create a simple vhdx file at the given path using
// default values.
//
//revive:disable-next-line:var-naming VHDX, not Vhdx
func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error {
params := CreateVirtualDiskParameters{
Version: 2,
@ -172,6 +175,8 @@ func DetachVirtualDisk(handle syscall.Handle) (err error) {
}
// DetachVhd detaches a vhd found at `path`.
//
//revive:disable-next-line:var-naming VHD, not Vhd
func DetachVhd(path string) error {
handle, err := OpenVirtualDisk(
path,
@ -181,12 +186,16 @@ func DetachVhd(path string) error {
if err != nil {
return err
}
defer syscall.CloseHandle(handle)
defer syscall.CloseHandle(handle) //nolint:errcheck
return DetachVirtualDisk(handle)
}
// AttachVirtualDisk attaches a virtual hard disk for use.
func AttachVirtualDisk(handle syscall.Handle, attachVirtualDiskFlag AttachVirtualDiskFlag, parameters *AttachVirtualDiskParameters) (err error) {
func AttachVirtualDisk(
handle syscall.Handle,
attachVirtualDiskFlag AttachVirtualDiskFlag,
parameters *AttachVirtualDiskParameters,
) (err error) {
// Supports both version 1 and 2 of the attach parameters as version 2 wasn't present in RS5.
if err := attachVirtualDisk(
handle,
@ -203,6 +212,8 @@ func AttachVirtualDisk(handle syscall.Handle, attachVirtualDiskFlag AttachVirtua
// AttachVhd attaches a virtual hard disk at `path` for use. Attaches using version 2
// of the ATTACH_VIRTUAL_DISK_PARAMETERS.
//
//revive:disable-next-line:var-naming VHD, not Vhd
func AttachVhd(path string) (err error) {
handle, err := OpenVirtualDisk(
path,
@ -213,7 +224,7 @@ func AttachVhd(path string) (err error) {
return err
}
defer syscall.CloseHandle(handle)
defer syscall.CloseHandle(handle) //nolint:errcheck
params := AttachVirtualDiskParameters{Version: 2}
if err := AttachVirtualDisk(
handle,
@ -226,7 +237,11 @@ func AttachVhd(path string) (err error) {
}
// OpenVirtualDisk obtains a handle to a VHD opened with supplied access mask and flags.
func OpenVirtualDisk(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask, openVirtualDiskFlags VirtualDiskFlag) (syscall.Handle, error) {
func OpenVirtualDisk(
vhdPath string,
virtualDiskAccessMask VirtualDiskAccessMask,
openVirtualDiskFlags VirtualDiskFlag,
) (syscall.Handle, error) {
parameters := OpenVirtualDiskParameters{Version: 2}
handle, err := OpenVirtualDiskWithParameters(
vhdPath,
@ -241,7 +256,12 @@ func OpenVirtualDisk(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask
}
// OpenVirtualDiskWithParameters obtains a handle to a VHD opened with supplied access mask, flags and parameters.
func OpenVirtualDiskWithParameters(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask, openVirtualDiskFlags VirtualDiskFlag, parameters *OpenVirtualDiskParameters) (syscall.Handle, error) {
func OpenVirtualDiskWithParameters(
vhdPath string,
virtualDiskAccessMask VirtualDiskAccessMask,
openVirtualDiskFlags VirtualDiskFlag,
parameters *OpenVirtualDiskParameters,
) (syscall.Handle, error) {
var (
handle syscall.Handle
defaultType VirtualStorageType
@ -279,7 +299,12 @@ func OpenVirtualDiskWithParameters(vhdPath string, virtualDiskAccessMask Virtual
}
// CreateVirtualDisk creates a virtual harddisk and returns a handle to the disk.
func CreateVirtualDisk(path string, virtualDiskAccessMask VirtualDiskAccessMask, createVirtualDiskFlags CreateVirtualDiskFlag, parameters *CreateVirtualDiskParameters) (syscall.Handle, error) {
func CreateVirtualDisk(
path string,
virtualDiskAccessMask VirtualDiskAccessMask,
createVirtualDiskFlags CreateVirtualDiskFlag,
parameters *CreateVirtualDiskParameters,
) (syscall.Handle, error) {
var (
handle syscall.Handle
defaultType VirtualStorageType
@ -323,6 +348,8 @@ func GetVirtualDiskPhysicalPath(handle syscall.Handle) (_ string, err error) {
}
// CreateDiffVhd is a helper function to create a differencing virtual disk.
//
//revive:disable-next-line:var-naming VHD, not Vhd
func CreateDiffVhd(diffVhdPath, baseVhdPath string, blockSizeInMB uint32) error {
// Setting `ParentPath` is how to signal to create a differencing disk.
createParams := &CreateVirtualDiskParameters{

View File

@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package vhd

View File

@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package winio
@ -47,9 +49,11 @@ var (
procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW")
procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW")
procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
procConvertStringSidToSidW = modadvapi32.NewProc("ConvertStringSidToSidW")
procGetSecurityDescriptorLength = modadvapi32.NewProc("GetSecurityDescriptorLength")
procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf")
procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW")
procLookupAccountSidW = modadvapi32.NewProc("LookupAccountSidW")
procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW")
procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW")
procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW")
@ -74,7 +78,6 @@ var (
procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult")
procbind = modws2_32.NewProc("bind")
)
func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
@ -123,6 +126,14 @@ func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision
return
}
func convertStringSidToSid(str *uint16, sid **byte) (err error) {
r1, _, e1 := syscall.Syscall(procConvertStringSidToSidW.Addr(), 2, uintptr(unsafe.Pointer(str)), uintptr(unsafe.Pointer(sid)), 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getSecurityDescriptorLength(sd uintptr) (len uint32) {
r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0)
len = uint32(r0)
@ -154,6 +165,14 @@ func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidS
return
}
func lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
r1, _, e1 := syscall.Syscall9(procLookupAccountSidW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(systemName)
@ -380,25 +399,25 @@ func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err erro
return
}
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) {
r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) {
func rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) {
r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) {
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) {
r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlNtStatusToDosError(status ntstatus) (winerr error) {
func rtlNtStatusToDosError(status ntStatus) (winerr error) {
r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
if r0 != 0 {
winerr = syscall.Errno(r0)
@ -417,11 +436,3 @@ func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint
}
return
}
func bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}

View File

@ -27,40 +27,40 @@
// The vast majority of use cases should simply use the match function with
// user input. The first step is to parse a specifier into a matcher:
//
// m, err := Parse("linux")
// if err != nil { ... }
// m, err := Parse("linux")
// if err != nil { ... }
//
// Once you have a matcher, use it to match against the platform declared by a
// component, typically from an image or runtime. Since extracting an images
// platform is a little more involved, we'll use an example against the
// platform default:
//
// if ok := m.Match(Default()); !ok { /* doesn't match */ }
// if ok := m.Match(Default()); !ok { /* doesn't match */ }
//
// This can be composed in loops for resolving runtimes or used as a filter for
// fetch and select images.
//
// More details of the specifier syntax and platform spec follow.
//
// Declaring Platform Support
// # Declaring Platform Support
//
// Components that have strict platform requirements should use the OCI
// platform specification to declare their support. Typically, this will be
// images and runtimes that should make these declaring which platform they
// support specifically. This looks roughly as follows:
//
// type Platform struct {
// Architecture string
// OS string
// Variant string
// }
// type Platform struct {
// Architecture string
// OS string
// Variant string
// }
//
// Most images and runtimes should at least set Architecture and OS, according
// to their GOARCH and GOOS values, respectively (follow the OCI image
// specification when in doubt). ARM should set variant under certain
// discussions, which are outlined below.
//
// Platform Specifiers
// # Platform Specifiers
//
// While the OCI platform specifications provide a tool for components to
// specify structured information, user input typically doesn't need the full
@ -77,7 +77,7 @@
// where the architecture may be known but a runtime may support images from
// different operating systems.
//
// Normalization
// # Normalization
//
// Because not all users are familiar with the way the Go runtime represents
// platforms, several normalizations have been provided to make this package
@ -85,17 +85,17 @@
//
// The following are performed for architectures:
//
// Value Normalized
// aarch64 arm64
// armhf arm
// armel arm/v6
// i386 386
// x86_64 amd64
// x86-64 amd64
// Value Normalized
// aarch64 arm64
// armhf arm
// armel arm/v6
// i386 386
// x86_64 amd64
// x86-64 amd64
//
// We also normalize the operating system `macos` to `darwin`.
//
// ARM Support
// # ARM Support
//
// To qualify ARM architecture, the Variant field is used to qualify the arm
// version. The most common arm version, v7, is represented without the variant

View File

@ -322,7 +322,7 @@ in_podman_task:
image_build_task: &image-build
name: "Build multi-arch $CTXDIR"
name: "Build multi-arch $FLAVOR"
alias: image_build
# Some of these container images take > 1h to build, limit
# this task to a specific Cirrus-Cron entry with this name.
@ -337,11 +337,11 @@ image_build_task: &image-build
type: "n2-standard-4"
matrix:
- env:
CTXDIR: contrib/buildahimage/upstream
FLAVOR: upstream
- env:
CTXDIR: contrib/buildahimage/testing
FLAVOR: testing
- env:
CTXDIR: contrib/buildahimage/stable
FLAVOR: stable
env:
DISTRO_NV: "${FEDORA_NAME}" # Required for repo cache extraction
BUILDAH_USERNAME: ENCRYPTED[70e1d4f026cba5d82fc067944baab10f7c71c64bb6b75fce4eeb5c106694b3bbc8e08f8a1b926d6e03e85cf4e21833bb]
@ -350,7 +350,7 @@ image_build_task: &image-build
CONTAINERS_PASSWORD: ENCRYPTED[886cf4cc126e50b2fd7f2792235a22bb79e4b81db43f803a6214a38d3fd6c04cd4e64570b562cb32b04e5fbc435404b6]
main_script:
- source /etc/automation_environment
- main.sh $CIRRUS_REPO_CLONE_URL $CTXDIR
- main.sh $CIRRUS_REPO_CLONE_URL contrib/buildahimage $FLAVOR
test_image_build_task:

View File

@ -123,7 +123,7 @@ validate: install.tools
.PHONY: install.tools
install.tools:
make -C tests/tools
$(MAKE) -C tests/tools
.PHONY: runc
runc: gopath
@ -146,7 +146,8 @@ install.cni.sudo: gopath
.PHONY: install
install:
install -D -m0755 bin/buildah $(DESTDIR)/$(BINDIR)/buildah
install -d -m 755 $(DESTDIR)/$(BINDIR)
install -m 755 bin/buildah $(DESTDIR)/$(BINDIR)/buildah
$(MAKE) -C docs install
.PHONY: uninstall
@ -157,7 +158,8 @@ uninstall:
.PHONY: install.completions
install.completions:
install -m 644 -D contrib/completions/bash/buildah $(DESTDIR)/$(BASHINSTALLDIR)/buildah
install -m 755 -d $(DESTDIR)/$(BASHINSTALLDIR)
install -m 644 contrib/completions/bash/buildah $(DESTDIR)/$(BASHINSTALLDIR)/buildah
.PHONY: install.runc
install.runc:

View File

@ -59,7 +59,7 @@ BuildRequires: btrfs-progs-devel
%if 0%{?fedora} <= 35
Requires: containers-common >= 4:1-39
%else
Requires: containers-common >= 4:1-46
Requires: containers-common-extra
%endif
%if 0%{?rhel}
BuildRequires: libseccomp-devel

View File

@ -30,7 +30,7 @@ const (
Package = "buildah"
// Version for the Package. Bump version in contrib/rpm/buildah.spec
// too.
Version = "1.28.0"
Version = "1.28.1-dev"
// DefaultRuntime if containers.conf fails.
DefaultRuntime = "runc"

View File

@ -34,7 +34,6 @@ import (
v1 "github.com/opencontainers/image-spec/specs-go/v1"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/openshift/imagebuilder"
"github.com/openshift/imagebuilder/dockerfile/parser"
"github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
)
@ -412,8 +411,6 @@ func buildDockerfilesOnce(ctx context.Context, store storage.Store, logger *logr
}
}
warnOnUnsetBuildArgs(logger, mainNode, options.Args)
for i, d := range dockerfilecontents[1:] {
additionalNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(d))
if err != nil {
@ -475,31 +472,6 @@ func buildDockerfilesOnce(ctx context.Context, store storage.Store, logger *logr
return exec.Build(ctx, stages)
}
func warnOnUnsetBuildArgs(logger *logrus.Logger, node *parser.Node, args map[string]string) {
argFound := make(map[string]bool)
for _, child := range node.Children {
switch strings.ToUpper(child.Value) {
case "ARG":
argName := child.Next.Value
if strings.Contains(argName, "=") {
res := strings.Split(argName, "=")
if res[1] != "" {
argFound[res[0]] = true
}
}
argHasValue := true
if !strings.Contains(argName, "=") {
argHasValue = argFound[argName]
}
if _, ok := args[argName]; !argHasValue && !ok {
logger.Warnf("missing %q build argument. Try adding %q to the command line", argName, fmt.Sprintf("--build-arg %s=<VALUE>", argName))
}
default:
continue
}
}
}
// preprocessContainerfileContents runs CPP(1) in preprocess-only mode on the input
// dockerfile content and will use ctxDir as the base include path.
func preprocessContainerfileContents(logger *logrus.Logger, containerfile string, r io.Reader, ctxDir string, cppFlags []string) (stdout io.Reader, err error) {

View File

@ -111,6 +111,7 @@ type Executor struct {
blobDirectory string
excludes []string
ignoreFile string
args map[string]string
unusedArgs map[string]struct{}
capabilities []string
devices define.ContainerDevices
@ -216,6 +217,7 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o
}
exec := Executor{
args: options.Args,
cacheFrom: options.CacheFrom,
cacheTo: options.CacheTo,
cacheTTL: options.CacheTTL,
@ -537,6 +539,45 @@ func markDependencyStagesForTarget(dependencyMap map[string]*stageDependencyInfo
}
}
func (b *Executor) warnOnUnsetBuildArgs(stages imagebuilder.Stages, dependencyMap map[string]*stageDependencyInfo, args map[string]string) {
argFound := make(map[string]bool)
for _, stage := range stages {
node := stage.Node // first line
for node != nil { // each line
for _, child := range node.Children {
switch strings.ToUpper(child.Value) {
case "ARG":
argName := child.Next.Value
if strings.Contains(argName, "=") {
res := strings.Split(argName, "=")
if res[1] != "" {
argFound[res[0]] = true
}
}
argHasValue := true
if !strings.Contains(argName, "=") {
argHasValue = argFound[argName]
}
if _, ok := args[argName]; !argHasValue && !ok {
shouldWarn := true
if stageDependencyInfo, ok := dependencyMap[stage.Name]; ok {
if !stageDependencyInfo.NeededByTarget && b.skipUnusedStages != types.OptionalBoolFalse {
shouldWarn = false
}
}
if shouldWarn {
b.logger.Warnf("missing %q build argument. Try adding %q to the command line", argName, fmt.Sprintf("--build-arg %s=<VALUE>", argName))
}
}
default:
continue
}
}
node = node.Next
}
}
}
// Build takes care of the details of running Prepare/Execute/Commit/Delete
// over each of the one or more parsed Dockerfiles and stages.
func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (imageID string, ref reference.Canonical, err error) {
@ -716,6 +757,8 @@ func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (image
// add base to current stage's dependency tree
// but also confirm if this is not in additional context.
if _, ok := b.additionalBuildContexts[mountFrom]; !ok {
// Treat from as a rootfs we need to preserve
b.rootfsMap[mountFrom] = true
if _, ok := dependencyMap[mountFrom]; ok {
// update current stage's dependency info
currentStageInfo := dependencyMap[stage.Name]
@ -741,6 +784,7 @@ func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (image
markDependencyStagesForTarget(dependencyMap, stage.Name)
}
}
b.warnOnUnsetBuildArgs(stages, dependencyMap, b.args)
type Result struct {
Index int

View File

@ -1302,6 +1302,24 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
if err != nil {
return "", nil, fmt.Errorf("checking if cached image exists from a previous build: %w", err)
}
// All the best effort to find cache on localstorage have failed try pulling
// cache from remote repo if `--cache-from` was configured and cacheKey was
// generated again after adding content summary.
if cacheID == "" && s.executor.cacheFrom != nil {
// only attempt to use cache again if pulling was successful
// otherwise do nothing and attempt to run the step, err != nil
// is ignored and will be automatically logged for --log-level debug
if id, err := s.pullCache(ctx, cacheKey); id != "" && err == nil {
logCachePulled(cacheKey)
cacheID, err = s.intermediateImageExists(ctx, node, addedContentSummary, s.stepRequiresLayer(step))
if err != nil {
return "", nil, fmt.Errorf("checking if cached image exists from a previous build: %w", err)
}
if cacheID != "" {
pulledAndUsedCacheImage = true
}
}
}
}
} else {
// This log line is majorly here so we can verify in tests

View File

@ -33,6 +33,9 @@ const (
BuildahCacheDir = "buildah-cache"
// mount=type=cache allows users to lock a cache store while its being used by another build
BuildahCacheLockfile = "buildah-cache-lockfile"
// All the lockfiles are stored in a separate directory inside `BuildahCacheDir`
// Example `/var/tmp/buildah-cache/<target>/buildah-cache-lockfile`
BuildahCacheLockfileDir = "buildah-cache-lockfiles"
)
var (
@ -184,10 +187,12 @@ func GetBindMount(ctx *types.SystemContext, args []string, contextDir string, st
}
// GetCacheMount parses a single cache mount entry from the --mount flag.
func GetCacheMount(args []string, store storage.Store, imageMountLabel string, additionalMountPoints map[string]internal.StageMountDetails) (specs.Mount, []string, error) {
//
// If this function succeeds and returns a non-nil lockfile.Locker, the caller must unlock it (when??).
func GetCacheMount(args []string, store storage.Store, imageMountLabel string, additionalMountPoints map[string]internal.StageMountDetails) (specs.Mount, lockfile.Locker, error) {
var err error
var mode uint64
lockedTargets := make([]string, 0)
var buildahLockFilesDir string
var (
setDest bool
setShared bool
@ -235,59 +240,59 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
sharing = kv[1]
case "bind-propagation":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
newMount.Options = append(newMount.Options, kv[1])
case "id":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
id = kv[1]
case "from":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
fromStage = kv[1]
case "target", "dst", "destination":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
if err := parse.ValidateVolumeCtrDir(kv[1]); err != nil {
return newMount, lockedTargets, err
return newMount, nil, err
}
newMount.Destination = kv[1]
setDest = true
case "src", "source":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
newMount.Source = kv[1]
case "mode":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
mode, err = strconv.ParseUint(kv[1], 8, 32)
if err != nil {
return newMount, lockedTargets, fmt.Errorf("unable to parse cache mode: %w", err)
return newMount, nil, fmt.Errorf("unable to parse cache mode: %w", err)
}
case "uid":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
uid, err = strconv.Atoi(kv[1])
if err != nil {
return newMount, lockedTargets, fmt.Errorf("unable to parse cache uid: %w", err)
return newMount, nil, fmt.Errorf("unable to parse cache uid: %w", err)
}
case "gid":
if len(kv) == 1 {
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadOptionArg)
}
gid, err = strconv.Atoi(kv[1])
if err != nil {
return newMount, lockedTargets, fmt.Errorf("unable to parse cache gid: %w", err)
return newMount, nil, fmt.Errorf("unable to parse cache gid: %w", err)
}
default:
return newMount, lockedTargets, fmt.Errorf("%v: %w", kv[0], errBadMntOption)
return newMount, nil, fmt.Errorf("%v: %w", kv[0], errBadMntOption)
}
}
@ -298,7 +303,7 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
}
if !setDest {
return newMount, lockedTargets, errBadVolDest
return newMount, nil, errBadVolDest
}
if fromStage != "" {
@ -315,7 +320,7 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
// Cache does not supports using image so if not stage found
// return with error
if mountPoint == "" {
return newMount, lockedTargets, fmt.Errorf("no stage found with name %s", fromStage)
return newMount, nil, fmt.Errorf("no stage found with name %s", fromStage)
}
// path should be /contextDir/specified path
newMount.Source = filepath.Join(mountPoint, filepath.Clean(string(filepath.Separator)+newMount.Source))
@ -331,13 +336,15 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
// create cache on host if not present
err = os.MkdirAll(cacheParent, os.FileMode(0755))
if err != nil {
return newMount, lockedTargets, fmt.Errorf("unable to create build cache directory: %w", err)
return newMount, nil, fmt.Errorf("unable to create build cache directory: %w", err)
}
if id != "" {
newMount.Source = filepath.Join(cacheParent, filepath.Clean(id))
buildahLockFilesDir = filepath.Join(BuildahCacheLockfileDir, filepath.Clean(id))
} else {
newMount.Source = filepath.Join(cacheParent, filepath.Clean(newMount.Destination))
buildahLockFilesDir = filepath.Join(BuildahCacheLockfileDir, filepath.Clean(newMount.Destination))
}
idPair := idtools.IDPair{
UID: uid,
@ -346,26 +353,40 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
//buildkit parity: change uid and gid if specified otheriwise keep `0`
err = idtools.MkdirAllAndChownNew(newMount.Source, os.FileMode(mode), idPair)
if err != nil {
return newMount, lockedTargets, fmt.Errorf("unable to change uid,gid of cache directory: %w", err)
return newMount, nil, fmt.Errorf("unable to change uid,gid of cache directory: %w", err)
}
// create a subdirectory inside `cacheParent` just to store lockfiles
buildahLockFilesDir = filepath.Join(cacheParent, buildahLockFilesDir)
err = os.MkdirAll(buildahLockFilesDir, os.FileMode(0700))
if err != nil {
return newMount, nil, fmt.Errorf("unable to create build cache lockfiles directory: %w", err)
}
}
var targetLock lockfile.Locker // = nil
succeeded := false
defer func() {
if !succeeded && targetLock != nil {
targetLock.Unlock()
}
}()
switch sharing {
case "locked":
// lock parent cache
lockfile, err := lockfile.GetLockfile(filepath.Join(newMount.Source, BuildahCacheLockfile))
lockfile, err := lockfile.GetLockfile(filepath.Join(buildahLockFilesDir, BuildahCacheLockfile))
if err != nil {
return newMount, lockedTargets, fmt.Errorf("unable to acquire lock when sharing mode is locked: %w", err)
return newMount, nil, fmt.Errorf("unable to acquire lock when sharing mode is locked: %w", err)
}
// Will be unlocked after the RUN step is executed.
lockfile.Lock()
lockedTargets = append(lockedTargets, filepath.Join(newMount.Source, BuildahCacheLockfile))
targetLock = lockfile
case "shared":
// do nothing since default is `shared`
break
default:
// error out for unknown values
return newMount, lockedTargets, fmt.Errorf("unrecognized value %q for field `sharing`: %w", sharing, err)
return newMount, nil, fmt.Errorf("unrecognized value %q for field `sharing`: %w", sharing, err)
}
// buildkit parity: default sharing should be shared
@ -383,11 +404,12 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
opts, err := parse.ValidateVolumeOpts(newMount.Options)
if err != nil {
return newMount, lockedTargets, err
return newMount, nil, err
}
newMount.Options = opts
return newMount, lockedTargets, nil
succeeded = true
return newMount, targetLock, nil
}
// ValidateVolumeMountHostDir validates the host path of buildah --volume
@ -474,19 +496,34 @@ func Volume(volume string) (specs.Mount, error) {
return mount, nil
}
// GetVolumes gets the volumes from --volume and --mount
func GetVolumes(ctx *types.SystemContext, store storage.Store, volumes []string, mounts []string, contextDir string) ([]specs.Mount, []string, []string, error) {
unifiedMounts, mountedImages, lockedTargets, err := getMounts(ctx, store, mounts, contextDir)
if err != nil {
return nil, mountedImages, lockedTargets, err
// UnlockLockArray is a helper for cleaning up after GetVolumes and the like.
func UnlockLockArray(locks []lockfile.Locker) {
for _, lock := range locks {
lock.Unlock()
}
}
// GetVolumes gets the volumes from --volume and --mount
//
// If this function succeeds, the caller must unlock the returned lockfile.Lockers if any (when??).
func GetVolumes(ctx *types.SystemContext, store storage.Store, volumes []string, mounts []string, contextDir string) ([]specs.Mount, []string, []lockfile.Locker, error) {
unifiedMounts, mountedImages, targetLocks, err := getMounts(ctx, store, mounts, contextDir)
if err != nil {
return nil, mountedImages, nil, err
}
succeeded := false
defer func() {
if !succeeded {
UnlockLockArray(targetLocks)
}
}()
volumeMounts, err := getVolumeMounts(volumes)
if err != nil {
return nil, mountedImages, lockedTargets, err
return nil, mountedImages, nil, err
}
for dest, mount := range volumeMounts {
if _, ok := unifiedMounts[dest]; ok {
return nil, mountedImages, lockedTargets, fmt.Errorf("%v: %w", dest, errDuplicateDest)
return nil, mountedImages, nil, fmt.Errorf("%v: %w", dest, errDuplicateDest)
}
unifiedMounts[dest] = mount
}
@ -495,19 +532,28 @@ func GetVolumes(ctx *types.SystemContext, store storage.Store, volumes []string,
for _, mount := range unifiedMounts {
finalMounts = append(finalMounts, mount)
}
return finalMounts, mountedImages, lockedTargets, nil
succeeded = true
return finalMounts, mountedImages, targetLocks, nil
}
// getMounts takes user-provided input from the --mount flag and creates OCI
// spec mounts.
// buildah run --mount type=bind,src=/etc/resolv.conf,target=/etc/resolv.conf ...
// buildah run --mount type=tmpfs,target=/dev/shm ...
func getMounts(ctx *types.SystemContext, store storage.Store, mounts []string, contextDir string) (map[string]specs.Mount, []string, []string, error) {
//
// If this function succeeds, the caller must unlock the returned lockfile.Lockers if any (when??).
func getMounts(ctx *types.SystemContext, store storage.Store, mounts []string, contextDir string) (map[string]specs.Mount, []string, []lockfile.Locker, error) {
// If `type` is not set default to "bind"
mountType := TypeBind
finalMounts := make(map[string]specs.Mount)
mountedImages := make([]string, 0)
lockedTargets := make([]string, 0)
targetLocks := make([]lockfile.Locker, 0)
succeeded := false
defer func() {
if !succeeded {
UnlockLockArray(targetLocks)
}
}()
errInvalidSyntax := errors.New("incorrect mount format: should be --mount type=<bind|tmpfs>,[src=<host-dir>,]target=<ctr-dir>[,options]")
@ -517,13 +563,13 @@ func getMounts(ctx *types.SystemContext, store storage.Store, mounts []string, c
for _, mount := range mounts {
tokens := strings.Split(mount, ",")
if len(tokens) < 2 {
return nil, mountedImages, lockedTargets, fmt.Errorf("%q: %w", mount, errInvalidSyntax)
return nil, mountedImages, nil, fmt.Errorf("%q: %w", mount, errInvalidSyntax)
}
for _, field := range tokens {
if strings.HasPrefix(field, "type=") {
kv := strings.Split(field, "=")
if len(kv) != 2 {
return nil, mountedImages, lockedTargets, fmt.Errorf("%q: %w", mount, errInvalidSyntax)
return nil, mountedImages, nil, fmt.Errorf("%q: %w", mount, errInvalidSyntax)
}
mountType = kv[1]
}
@ -532,38 +578,41 @@ func getMounts(ctx *types.SystemContext, store storage.Store, mounts []string, c
case TypeBind:
mount, image, err := GetBindMount(ctx, tokens, contextDir, store, "", nil)
if err != nil {
return nil, mountedImages, lockedTargets, err
return nil, mountedImages, nil, err
}
if _, ok := finalMounts[mount.Destination]; ok {
return nil, mountedImages, lockedTargets, fmt.Errorf("%v: %w", mount.Destination, errDuplicateDest)
return nil, mountedImages, nil, fmt.Errorf("%v: %w", mount.Destination, errDuplicateDest)
}
finalMounts[mount.Destination] = mount
mountedImages = append(mountedImages, image)
case TypeCache:
mount, lockedPaths, err := GetCacheMount(tokens, store, "", nil)
lockedTargets = lockedPaths
mount, tl, err := GetCacheMount(tokens, store, "", nil)
if err != nil {
return nil, mountedImages, lockedTargets, err
return nil, mountedImages, nil, err
}
if tl != nil {
targetLocks = append(targetLocks, tl)
}
if _, ok := finalMounts[mount.Destination]; ok {
return nil, mountedImages, lockedTargets, fmt.Errorf("%v: %w", mount.Destination, errDuplicateDest)
return nil, mountedImages, nil, fmt.Errorf("%v: %w", mount.Destination, errDuplicateDest)
}
finalMounts[mount.Destination] = mount
case TypeTmpfs:
mount, err := GetTmpfsMount(tokens)
if err != nil {
return nil, mountedImages, lockedTargets, err
return nil, mountedImages, nil, err
}
if _, ok := finalMounts[mount.Destination]; ok {
return nil, mountedImages, lockedTargets, fmt.Errorf("%v: %w", mount.Destination, errDuplicateDest)
return nil, mountedImages, nil, fmt.Errorf("%v: %w", mount.Destination, errDuplicateDest)
}
finalMounts[mount.Destination] = mount
default:
return nil, mountedImages, lockedTargets, fmt.Errorf("invalid filesystem type %q", mountType)
return nil, mountedImages, nil, fmt.Errorf("invalid filesystem type %q", mountType)
}
}
return finalMounts, mountedImages, lockedTargets, nil
succeeded = true
return finalMounts, mountedImages, targetLocks, nil
}
// GetTmpfsMount parses a single tmpfs mount entry from the --mount flag

View File

@ -8,6 +8,7 @@ import (
"net"
"os"
"path/filepath"
"runtime"
"sync"
"time"
@ -68,6 +69,13 @@ func newAgentServerSocket(socketPath string) (*AgentServer, error) {
// Serve starts the SSH agent on the host and returns the path of the socket where the agent is serving
func (a *AgentServer) Serve(processLabel string) (string, error) {
// Calls to `selinux.SetSocketLabel` should be wrapped in
// runtime.LockOSThread()/runtime.UnlockOSThread() until
// the the socket is created to guarantee another goroutine
// does not migrate to the current thread before execution
// is complete.
// Ref: https://github.com/opencontainers/selinux/blob/main/go-selinux/selinux.go#L158
runtime.LockOSThread()
err := selinux.SetSocketLabel(processLabel)
if err != nil {
return "", err
@ -83,7 +91,12 @@ func (a *AgentServer) Serve(processLabel string) (string, error) {
if err != nil {
return "", err
}
// Reset socket label.
err = selinux.SetSocketLabel("")
// Unlock the thread only if the process label could be restored
// successfully. Otherwise leave the thread locked and the Go runtime
// will terminate it once it returns to the threads pool.
runtime.UnlockOSThread()
if err != nil {
return "", err
}

View File

@ -8,6 +8,7 @@ import (
"github.com/containers/buildah/internal"
"github.com/containers/buildah/pkg/sshagent"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/lockfile"
"github.com/opencontainers/runtime-spec/specs-go"
spec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
@ -176,8 +177,8 @@ type runMountArtifacts struct {
Agents []*sshagent.AgentServer
// SSHAuthSock is the path to the ssh auth sock inside the container
SSHAuthSock string
// LockedTargets to be unlocked if there are any.
LockedTargets []string
// TargetLocks to be unlocked if there are any.
TargetLocks []lockfile.Locker
}
// RunMountInfo are the available run mounts for this run

View File

@ -1256,6 +1256,7 @@ func init() {
reexec.Register(runUsingRuntimeCommand, runUsingRuntimeMain)
}
// If this succeeds, the caller must call cleanupMounts().
func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath string, optionMounts []specs.Mount, bindFiles map[string]string, builtinVolumes, volumeMounts []string, runFileMounts []string, runMountInfo runMountInfo) (*runMountArtifacts, error) {
// Start building a new list of mounts.
var mounts []specs.Mount
@ -1318,6 +1319,12 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st
if err != nil {
return nil, err
}
succeeded := false
defer func() {
if !succeeded {
internalParse.UnlockLockArray(mountArtifacts.TargetLocks)
}
}()
// Add temporary copies of the contents of volume locations at the
// volume locations, unless we already have something there.
builtins, err := runSetupBuiltinVolumes(b.MountLabel, mountPoint, cdir, builtinVolumes, int(rootUID), int(rootGID))
@ -1353,6 +1360,7 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st
// Set the list in the spec.
spec.Mounts = mounts
succeeded = true
return mountArtifacts, nil
}
@ -1444,6 +1452,8 @@ func cleanableDestinationListFromMounts(mounts []spec.Mount) []string {
}
// runSetupRunMounts sets up mounts that exist only in this RUN, not in subsequent runs
//
// If this function succeeds, the caller must unlock runMountArtifacts.TargetLocks (when??)
func (b *Builder) runSetupRunMounts(mounts []string, sources runMountInfo, idMaps IDMaps) ([]spec.Mount, *runMountArtifacts, error) {
// If `type` is not set default to "bind"
mountType := internalParse.TypeBind
@ -1454,7 +1464,13 @@ func (b *Builder) runSetupRunMounts(mounts []string, sources runMountInfo, idMap
agents := make([]*sshagent.AgentServer, 0, len(mounts))
sshCount := 0
defaultSSHSock := ""
lockedTargets := []string{}
targetLocks := []lockfile.Locker{}
succeeded := false
defer func() {
if !succeeded {
internalParse.UnlockLockArray(targetLocks)
}
}()
for _, mount := range mounts {
tokens := strings.Split(mount, ",")
for _, field := range tokens {
@ -1513,24 +1529,27 @@ func (b *Builder) runSetupRunMounts(mounts []string, sources runMountInfo, idMap
finalMounts = append(finalMounts, *mount)
mountTargets = append(mountTargets, mount.Destination)
case "cache":
mount, lockedPaths, err := b.getCacheMount(tokens, sources.StageMountPoints, idMaps)
mount, tl, err := b.getCacheMount(tokens, sources.StageMountPoints, idMaps)
if err != nil {
return nil, nil, err
}
finalMounts = append(finalMounts, *mount)
mountTargets = append(mountTargets, mount.Destination)
lockedTargets = lockedPaths
if tl != nil {
targetLocks = append(targetLocks, tl)
}
default:
return nil, nil, fmt.Errorf("invalid mount type %q", mountType)
}
}
succeeded = true
artifacts := &runMountArtifacts{
RunMountTargets: mountTargets,
TmpFiles: tmpFiles,
Agents: agents,
MountedImages: mountImages,
SSHAuthSock: defaultSSHSock,
LockedTargets: lockedTargets,
TargetLocks: targetLocks,
}
return finalMounts, artifacts, nil
}
@ -1862,30 +1881,6 @@ func (b *Builder) cleanupRunMounts(context *imageTypes.SystemContext, mountpoint
}
}
// unlock if any locked files from this RUN statement
for _, path := range artifacts.LockedTargets {
_, err := os.Stat(path)
if err != nil {
// Lockfile not found this might be a problem,
// since LockedTargets must contain list of all locked files
// don't break here since we need to unlock other files but
// log so user can take a look
logrus.Warnf("Lockfile %q was expected here, stat failed with %v", path, err)
continue
}
lockfile, err := lockfile.GetLockfile(path)
if err != nil {
// unable to get lockfile
// lets log error and continue
// unlocking other files
logrus.Warn(err)
continue
}
if lockfile.Locked() {
lockfile.Unlock()
} else {
logrus.Warnf("Lockfile %q was expected to be locked, this is unexpected", path)
continue
}
}
internalParse.UnlockLockArray(artifacts.TargetLocks)
return prevErr
}

View File

@ -25,6 +25,7 @@ import (
nettypes "github.com/containers/common/libnetwork/types"
"github.com/containers/common/pkg/config"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/lockfile"
"github.com/containers/storage/pkg/stringid"
"github.com/docker/go-units"
"github.com/opencontainers/runtime-spec/specs-go"
@ -308,7 +309,8 @@ func setupSpecialMountSpecChanges(spec *spec.Spec, shmSize string) ([]specs.Moun
return spec.Mounts, nil
}
func (b *Builder) getCacheMount(tokens []string, stageMountPoints map[string]internal.StageMountDetails, idMaps IDMaps) (*spec.Mount, []string, error) {
// If this function succeeds and returns a non-nil lockfile.Locker, the caller must unlock it (when??).
func (b *Builder) getCacheMount(tokens []string, stageMountPoints map[string]internal.StageMountDetails, idMaps IDMaps) (*spec.Mount, lockfile.Locker, error) {
return nil, nil, errors.New("cache mounts not supported on freebsd")
}

View File

@ -34,6 +34,7 @@ import (
hooksExec "github.com/containers/common/pkg/hooks/exec"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/ioutils"
"github.com/containers/storage/pkg/lockfile"
"github.com/containers/storage/pkg/stringid"
"github.com/containers/storage/pkg/unshare"
"github.com/docker/go-units"
@ -1196,18 +1197,26 @@ func checkIdsGreaterThan5(ids []spec.LinuxIDMapping) bool {
return false
}
func (b *Builder) getCacheMount(tokens []string, stageMountPoints map[string]internal.StageMountDetails, idMaps IDMaps) (*spec.Mount, []string, error) {
// If this function succeeds and returns a non-nil lockfile.Locker, the caller must unlock it (when??).
func (b *Builder) getCacheMount(tokens []string, stageMountPoints map[string]internal.StageMountDetails, idMaps IDMaps) (*spec.Mount, lockfile.Locker, error) {
var optionMounts []specs.Mount
mount, lockedTargets, err := internalParse.GetCacheMount(tokens, b.store, b.MountLabel, stageMountPoints)
mount, targetLock, err := internalParse.GetCacheMount(tokens, b.store, b.MountLabel, stageMountPoints)
if err != nil {
return nil, lockedTargets, err
return nil, nil, err
}
succeeded := false
defer func() {
if !succeeded && targetLock != nil {
targetLock.Unlock()
}
}()
optionMounts = append(optionMounts, mount)
volumes, err := b.runSetupVolumeMounts(b.MountLabel, nil, optionMounts, idMaps)
if err != nil {
return nil, lockedTargets, err
return nil, nil, err
}
return &volumes[0], lockedTargets, nil
succeeded = true
return &volumes[0], targetLock, nil
}
// setPdeathsig sets a parent-death signal for the process

View File

@ -46,6 +46,8 @@ type Image struct {
ociv1Image *ociv1.Image
// Names() parsed into references.
namesReferences []reference.Reference
// Calculating the Size() is expensive, so cache it.
size *int64
}
}
@ -62,6 +64,7 @@ func (i *Image) reload() error {
i.cached.completeInspectData = nil
i.cached.ociv1Image = nil
i.cached.namesReferences = nil
i.cached.size = nil
return nil
}
@ -775,8 +778,13 @@ func (i *Image) Unmount(force bool) error {
// Size computes the size of the image layers and associated data.
func (i *Image) Size() (int64, error) {
// TODO: cache the result to optimize performance of subsequent calls
return i.runtime.store.ImageSize(i.ID())
if i.cached.size != nil {
return *i.cached.size, nil
}
size, err := i.runtime.store.ImageSize(i.ID())
i.cached.size = &size
return size, err
}
// HasDifferentDigestOptions allows for customizing the check if another

View File

@ -170,6 +170,11 @@ func createMacvlan(network *types.Network) error {
}
}
// always turn dns off with macvlan, it is not implemented in netavark
// and makes little sense to support with macvlan
// see https://github.com/containers/netavark/pull/467
network.DNSEnabled = false
// we already validated the drivers before so we just have to set the default here
switch network.IPAMOptions[types.Driver] {
case "":

View File

@ -54,7 +54,7 @@ type Network struct {
// to public or other Networks.
Internal bool `json:"internal"`
// DNSEnabled is whether name resolution is active for container on
// this Network.
// this Network. Only supported with the bridge driver.
DNSEnabled bool `json:"dns_enabled"`
// Labels is a set of key-value labels that have been applied to the
// Network.

View File

@ -843,7 +843,7 @@ func (m *MachineConfig) URI() string {
func (c *EngineConfig) findRuntime() string {
// Search for crun first followed by runc, kata, runsc
for _, name := range []string{"crun", "runc", "runj", "kata", "runsc"} {
for _, name := range []string{"crun", "runc", "runj", "kata", "runsc", "ocijail"} {
for _, v := range c.OCIRuntimes[name] {
if _, err := os.Stat(v); err == nil {
return name

View File

@ -543,7 +543,7 @@ default_sysctls = [
# List of the OCI runtimes that support --format=json. When json is supported
# engine will use it for reporting nicer errors.
#
#runtime_supports_json = ["crun", "runc", "kata", "runsc", "krun"]
#runtime_supports_json = ["crun", "runc", "kata", "runsc", "youki", "krun"]
# List of the OCI runtimes that supports running containers with KVM Separation.
#
@ -654,6 +654,13 @@ default_sysctls = [
# "/run/current-system/sw/bin/runsc",
#]
#youki = [
# "/usr/local/bin/youki",
# "/usr/bin/youki",
# "/bin/youki",
# "/run/current-system/sw/bin/youki",
#]
#krun = [
# "/usr/bin/krun",
# "/usr/local/bin/krun",

View File

@ -485,7 +485,7 @@ default_sysctls = [
# List of the OCI runtimes that support --format=json. When json is supported
# engine will use it for reporting nicer errors.
#
#runtime_supports_json = ["crun", "runc", "kata", "runsc", "krun"]
#runtime_supports_json = ["crun", "runc", "kata", "runsc", "youki", "krun"]
# List of the OCI runtimes that supports running containers with KVM Separation.
#
@ -590,6 +590,13 @@ default_sysctls = [
# "/run/current-system/sw/bin/runsc",
#]
#youki = [
# "/usr/local/bin/youki",
# "/usr/bin/youki",
# "/bin/youki",
# "/run/current-system/sw/bin/youki",
#]
#krun = [
# "/usr/bin/krun",
# "/usr/local/bin/krun",

View File

@ -364,10 +364,19 @@ func defaultConfigFromMemory() (*EngineConfig, error) {
"/sbin/runsc",
"/run/current-system/sw/bin/runsc",
},
"youki": {
"/usr/local/bin/youki",
"/usr/bin/youki",
"/bin/youki",
"/run/current-system/sw/bin/youki",
},
"krun": {
"/usr/bin/krun",
"/usr/local/bin/krun",
},
"ocijail": {
"/usr/local/bin/ocijail",
},
}
// Needs to be called after populating c.OCIRuntimes.
c.OCIRuntime = c.findRuntime()
@ -401,6 +410,7 @@ func defaultConfigFromMemory() (*EngineConfig, error) {
"runc",
"kata",
"runsc",
"youki",
"krun",
}
c.RuntimeSupportsNoCgroups = []string{"crun", "krun"}

View File

@ -238,7 +238,7 @@ func (d *bpCompressionStepData) updateCompressionEdits(operation *types.LayerCom
}
// recordValidatedBlobData updates b.blobInfoCache with data about the created uploadedInfo adnd the original srcInfo.
// This must ONLY be called if all data has been validated by OUR code, and is not comming from third parties.
// This must ONLY be called if all data has been validated by OUR code, and is not coming from third parties.
func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInfo types.BlobInfo, srcInfo types.BlobInfo,
encryptionStep *bpEncryptionStepData, decryptionStep *bpDecryptionStepData) error {
// Dont record any associations that involve encrypted data. This is a bit crude,

View File

@ -64,7 +64,7 @@ func NewWriter(sys *types.SystemContext, path string) (*Writer, error) {
}, nil
}
// imageCommitted notifies the Writer that at least one image was successfully commited to the stream.
// imageCommitted notifies the Writer that at least one image was successfully committed to the stream.
func (w *Writer) imageCommitted() {
w.mutex.Lock()
defer w.mutex.Unlock()

View File

@ -30,13 +30,13 @@ func newDockerClient(sys *types.SystemContext) (*dockerclient.Client, error) {
//
// Similarly, if we want to communicate over plain HTTP on a TCP socket, we also need to set
// TLSClientConfig to nil. This can be achieved by using the form `http://`
url, err := dockerclient.ParseHostURL(host)
serverURL, err := dockerclient.ParseHostURL(host)
if err != nil {
return nil, err
}
var httpClient *http.Client
if url.Scheme != "unix" {
if url.Scheme == "http" {
if serverURL.Scheme != "unix" {
if serverURL.Scheme == "http" {
httpClient = httpConfig()
} else {
hc, err := tlsConfig(sys)

View File

@ -475,11 +475,11 @@ func (c *dockerClient) makeRequest(ctx context.Context, method, path string, hea
}
urlString := fmt.Sprintf("%s://%s%s", c.scheme, c.registry, path)
url, err := url.Parse(urlString)
requestURL, err := url.Parse(urlString)
if err != nil {
return nil, err
}
return c.makeRequestToResolvedURL(ctx, method, url, headers, stream, -1, auth, extraScope)
return c.makeRequestToResolvedURL(ctx, method, requestURL, headers, stream, -1, auth, extraScope)
}
// Checks if the auth headers in the response contain an indication of a failed
@ -542,11 +542,11 @@ func parseRetryAfter(res *http.Response, fallbackDelay time.Duration) time.Durat
// makeRequest should generally be preferred.
// In case of an HTTP 429 status code in the response, it may automatically retry a few times.
// TODO(runcom): too many arguments here, use a struct
func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method string, url *url.URL, headers map[string][]string, stream io.Reader, streamLen int64, auth sendAuth, extraScope *authScope) (*http.Response, error) {
func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method string, requestURL *url.URL, headers map[string][]string, stream io.Reader, streamLen int64, auth sendAuth, extraScope *authScope) (*http.Response, error) {
delay := backoffInitialDelay
attempts := 0
for {
res, err := c.makeRequestToResolvedURLOnce(ctx, method, url, headers, stream, streamLen, auth, extraScope)
res, err := c.makeRequestToResolvedURLOnce(ctx, method, requestURL, headers, stream, streamLen, auth, extraScope)
attempts++
// By default we use pre-defined scopes per operation. In
@ -567,7 +567,7 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method stri
// Note: This retry ignores extraScope. Thats, strictly speaking, incorrect, but we dont currently
// expect the insufficient_scope errors to happen for those callers. If that changes, we can add support
// for more than one extra scope.
res, err = c.makeRequestToResolvedURLOnce(ctx, method, url, headers, stream, streamLen, auth, newScope)
res, err = c.makeRequestToResolvedURLOnce(ctx, method, requestURL, headers, stream, streamLen, auth, newScope)
extraScope = newScope
}
}
@ -583,7 +583,7 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method stri
if delay > backoffMaxDelay {
delay = backoffMaxDelay
}
logrus.Debugf("Too many requests to %s: sleeping for %f seconds before next attempt", url.Redacted(), delay.Seconds())
logrus.Debugf("Too many requests to %s: sleeping for %f seconds before next attempt", requestURL.Redacted(), delay.Seconds())
select {
case <-ctx.Done():
return nil, ctx.Err()
@ -598,8 +598,8 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method stri
// streamLen, if not -1, specifies the length of the data expected on stream.
// makeRequest should generally be preferred.
// Note that no exponential back off is performed when receiving an http 429 status code.
func (c *dockerClient) makeRequestToResolvedURLOnce(ctx context.Context, method string, url *url.URL, headers map[string][]string, stream io.Reader, streamLen int64, auth sendAuth, extraScope *authScope) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, method, url.String(), stream)
func (c *dockerClient) makeRequestToResolvedURLOnce(ctx context.Context, method string, resolvedURL *url.URL, headers map[string][]string, stream io.Reader, streamLen int64, auth sendAuth, extraScope *authScope) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, method, resolvedURL.String(), stream)
if err != nil {
return nil, err
}
@ -618,7 +618,7 @@ func (c *dockerClient) makeRequestToResolvedURLOnce(ctx context.Context, method
return nil, err
}
}
logrus.Debugf("%s %s", method, url.Redacted())
logrus.Debugf("%s %s", method, resolvedURL.Redacted())
res, err := c.client.Do(req)
if err != nil {
return nil, err
@ -811,17 +811,17 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
c.client = &http.Client{Transport: tr}
ping := func(scheme string) error {
url, err := url.Parse(fmt.Sprintf(resolvedPingV2URL, scheme, c.registry))
pingURL, err := url.Parse(fmt.Sprintf(resolvedPingV2URL, scheme, c.registry))
if err != nil {
return err
}
resp, err := c.makeRequestToResolvedURL(ctx, http.MethodGet, url, nil, nil, -1, noAuth, nil)
resp, err := c.makeRequestToResolvedURL(ctx, http.MethodGet, pingURL, nil, nil, -1, noAuth, nil)
if err != nil {
logrus.Debugf("Ping %s err %s (%#v)", url.Redacted(), err.Error(), err)
logrus.Debugf("Ping %s err %s (%#v)", pingURL.Redacted(), err.Error(), err)
return err
}
defer resp.Body.Close()
logrus.Debugf("Ping %s status %d", url.Redacted(), resp.StatusCode)
logrus.Debugf("Ping %s status %d", pingURL.Redacted(), resp.StatusCode)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
return registryHTTPResponseToError(resp)
}
@ -841,17 +841,17 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
}
// best effort to understand if we're talking to a V1 registry
pingV1 := func(scheme string) bool {
url, err := url.Parse(fmt.Sprintf(resolvedPingV1URL, scheme, c.registry))
pingURL, err := url.Parse(fmt.Sprintf(resolvedPingV1URL, scheme, c.registry))
if err != nil {
return false
}
resp, err := c.makeRequestToResolvedURL(ctx, http.MethodGet, url, nil, nil, -1, noAuth, nil)
resp, err := c.makeRequestToResolvedURL(ctx, http.MethodGet, pingURL, nil, nil, -1, noAuth, nil)
if err != nil {
logrus.Debugf("Ping %s err %s (%#v)", url.Redacted(), err.Error(), err)
logrus.Debugf("Ping %s err %s (%#v)", pingURL.Redacted(), err.Error(), err)
return false
}
defer resp.Body.Close()
logrus.Debugf("Ping %s status %d", url.Redacted(), resp.StatusCode)
logrus.Debugf("Ping %s status %d", pingURL.Redacted(), resp.StatusCode)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
return false
}
@ -909,14 +909,14 @@ func (c *dockerClient) getExternalBlob(ctx context.Context, urls []string) (io.R
return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
}
for _, u := range urls {
url, err := url.Parse(u)
if err != nil || (url.Scheme != "http" && url.Scheme != "https") {
blobURL, err := url.Parse(u)
if err != nil || (blobURL.Scheme != "http" && blobURL.Scheme != "https") {
continue // unsupported url. skip this url.
}
// NOTE: we must not authenticate on additional URLs as those
// can be abused to leak credentials or tokens. Please
// refer to CVE-2020-15157 for more information.
resp, err = c.makeRequestToResolvedURL(ctx, http.MethodGet, url, nil, nil, -1, noAuth, nil)
resp, err = c.makeRequestToResolvedURL(ctx, http.MethodGet, blobURL, nil, nil, -1, noAuth, nil)
if err == nil {
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("error fetching external blob from %q: %d (%s)", u, resp.StatusCode, http.StatusText(resp.StatusCode))

View File

@ -407,7 +407,7 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context,
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
refTail := ""
var refTail string
if instanceDigest != nil {
// If the instanceDigest is provided, then use it as the refTail, because the reference,
// whether it includes a tag or a digest, refers to the list as a whole, and not this
@ -580,8 +580,8 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures []signature
// NOTE: Keep this in sync with docs/signature-protocols.md!
for i, signature := range signatures {
url := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
err := d.putOneSignature(url, signature)
sigURL := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
err := d.putOneSignature(sigURL, signature)
if err != nil {
return err
}
@ -592,8 +592,8 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures []signature
// is enough for dockerImageSource to stop looking for other signatures, so that
// is sufficient.
for i := len(signatures); ; i++ {
url := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
missing, err := d.c.deleteOneSignature(url)
sigURL := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
missing, err := d.c.deleteOneSignature(sigURL)
if err != nil {
return err
}
@ -605,13 +605,13 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures []signature
return nil
}
// putOneSignature stores sig to url.
// putOneSignature stores sig to sigURL.
// NOTE: Keep this in sync with docs/signature-protocols.md!
func (d *dockerImageDestination) putOneSignature(url *url.URL, sig signature.Signature) error {
switch url.Scheme {
func (d *dockerImageDestination) putOneSignature(sigURL *url.URL, sig signature.Signature) error {
switch sigURL.Scheme {
case "file":
logrus.Debugf("Writing to %s", url.Path)
err := os.MkdirAll(filepath.Dir(url.Path), 0755)
logrus.Debugf("Writing to %s", sigURL.Path)
err := os.MkdirAll(filepath.Dir(sigURL.Path), 0755)
if err != nil {
return err
}
@ -619,16 +619,16 @@ func (d *dockerImageDestination) putOneSignature(url *url.URL, sig signature.Sig
if err != nil {
return err
}
err = os.WriteFile(url.Path, blob, 0644)
err = os.WriteFile(sigURL.Path, blob, 0644)
if err != nil {
return err
}
return nil
case "http", "https":
return fmt.Errorf("Writing directly to a %s lookaside %s is not supported. Configure a lookaside-staging: location", url.Scheme, url.Redacted())
return fmt.Errorf("Writing directly to a %s lookaside %s is not supported. Configure a lookaside-staging: location", sigURL.Scheme, sigURL.Redacted())
default:
return fmt.Errorf("Unsupported scheme when writing signature to %s", url.Redacted())
return fmt.Errorf("Unsupported scheme when writing signature to %s", sigURL.Redacted())
}
}
@ -767,23 +767,23 @@ func (d *dockerImageDestination) putBlobBytesAsOCI(ctx context.Context, contents
}, nil
}
// deleteOneSignature deletes a signature from url, if it exists.
// deleteOneSignature deletes a signature from sigURL, if it exists.
// If it successfully determines that the signature does not exist, returns (true, nil)
// NOTE: Keep this in sync with docs/signature-protocols.md!
func (c *dockerClient) deleteOneSignature(url *url.URL) (missing bool, err error) {
switch url.Scheme {
func (c *dockerClient) deleteOneSignature(sigURL *url.URL) (missing bool, err error) {
switch sigURL.Scheme {
case "file":
logrus.Debugf("Deleting %s", url.Path)
err := os.Remove(url.Path)
logrus.Debugf("Deleting %s", sigURL.Path)
err := os.Remove(sigURL.Path)
if err != nil && os.IsNotExist(err) {
return true, nil
}
return false, err
case "http", "https":
return false, fmt.Errorf("Writing directly to a %s lookaside %s is not supported. Configure a lookaside-staging: location", url.Scheme, url.Redacted())
return false, fmt.Errorf("Writing directly to a %s lookaside %s is not supported. Configure a lookaside-staging: location", sigURL.Scheme, sigURL.Redacted())
default:
return false, fmt.Errorf("Unsupported scheme when deleting signature from %s", url.Redacted())
return false, fmt.Errorf("Unsupported scheme when deleting signature from %s", sigURL.Redacted())
}
}

View File

@ -456,8 +456,8 @@ func (s *dockerImageSource) getSignaturesFromLookaside(ctx context.Context, inst
return nil, fmt.Errorf("server provided %d signatures, assuming that's unreasonable and a server error", maxLookasideSignatures)
}
url := lookasideStorageURL(s.c.signatureBase, manifestDigest, i)
signature, missing, err := s.getOneSignature(ctx, url)
sigURL := lookasideStorageURL(s.c.signatureBase, manifestDigest, i)
signature, missing, err := s.getOneSignature(ctx, sigURL)
if err != nil {
return nil, err
}
@ -469,14 +469,14 @@ func (s *dockerImageSource) getSignaturesFromLookaside(ctx context.Context, inst
return signatures, nil
}
// getOneSignature downloads one signature from url, and returns (signature, false, nil)
// getOneSignature downloads one signature from sigURL, and returns (signature, false, nil)
// If it successfully determines that the signature does not exist, returns (nil, true, nil).
// NOTE: Keep this in sync with docs/signature-protocols.md!
func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (signature.Signature, bool, error) {
switch url.Scheme {
func (s *dockerImageSource) getOneSignature(ctx context.Context, sigURL *url.URL) (signature.Signature, bool, error) {
switch sigURL.Scheme {
case "file":
logrus.Debugf("Reading %s", url.Path)
sigBlob, err := os.ReadFile(url.Path)
logrus.Debugf("Reading %s", sigURL.Path)
sigBlob, err := os.ReadFile(sigURL.Path)
if err != nil {
if os.IsNotExist(err) {
return nil, true, nil
@ -485,13 +485,13 @@ func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (
}
sig, err := signature.FromBlob(sigBlob)
if err != nil {
return nil, false, fmt.Errorf("parsing signature %q: %w", url.Path, err)
return nil, false, fmt.Errorf("parsing signature %q: %w", sigURL.Path, err)
}
return sig, false, nil
case "http", "https":
logrus.Debugf("GET %s", url.Redacted())
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
logrus.Debugf("GET %s", sigURL.Redacted())
req, err := http.NewRequestWithContext(ctx, http.MethodGet, sigURL.String(), nil)
if err != nil {
return nil, false, err
}
@ -504,12 +504,12 @@ func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (
logrus.Debugf("... got status 404, as expected = end of signatures")
return nil, true, nil
} else if res.StatusCode != http.StatusOK {
return nil, false, fmt.Errorf("reading signature from %s: status %d (%s)", url.Redacted(), res.StatusCode, http.StatusText(res.StatusCode))
return nil, false, fmt.Errorf("reading signature from %s: status %d (%s)", sigURL.Redacted(), res.StatusCode, http.StatusText(res.StatusCode))
}
contentType := res.Header.Get("Content-Type")
if mimeType := simplifyContentType(contentType); mimeType == "text/html" {
logrus.Warnf("Signature %q has Content-Type %q, unexpected for a signature", url.Redacted(), contentType)
logrus.Warnf("Signature %q has Content-Type %q, unexpected for a signature", sigURL.Redacted(), contentType)
// Dont immediately fail; the lookaside spec does not place any requirements on Content-Type.
// If the content really is HTML, its going to fail in signature.FromBlob.
}
@ -520,12 +520,12 @@ func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (
}
sig, err := signature.FromBlob(sigBlob)
if err != nil {
return nil, false, fmt.Errorf("parsing signature %s: %w", url.Redacted(), err)
return nil, false, fmt.Errorf("parsing signature %s: %w", sigURL.Redacted(), err)
}
return sig, false, nil
default:
return nil, false, fmt.Errorf("Unsupported scheme when reading signature from %s", url.Redacted())
return nil, false, fmt.Errorf("Unsupported scheme when reading signature from %s", sigURL.Redacted())
}
}
@ -649,8 +649,8 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere
}
for i := 0; ; i++ {
url := lookasideStorageURL(c.signatureBase, manifestDigest, i)
missing, err := c.deleteOneSignature(url)
sigURL := lookasideStorageURL(c.signatureBase, manifestDigest, i)
missing, err := c.deleteOneSignature(sigURL)
if err != nil {
return err
}

View File

@ -163,17 +163,17 @@ func loadAndMergeConfig(dirPath string) (*registryConfiguration, error) {
// the usage of the BaseURL is defined under docker/distribution registries—separate storage of docs/signature-protocols.md
func (config *registryConfiguration) lookasideStorageBaseURL(dr dockerReference, write bool) (*url.URL, error) {
topLevel := config.signatureTopLevel(dr, write)
var url *url.URL
var baseURL *url.URL
if topLevel != "" {
u, err := url.Parse(topLevel)
if err != nil {
return nil, fmt.Errorf("Invalid signature storage URL %s: %w", topLevel, err)
}
url = u
baseURL = u
} else {
// returns default directory if no lookaside specified in configuration file
url = builtinDefaultLookasideStorageDir(rootless.GetRootlessEUID())
logrus.Debugf(" No signature storage configuration found for %s, using built-in default %s", dr.PolicyConfigurationIdentity(), url.Redacted())
baseURL = builtinDefaultLookasideStorageDir(rootless.GetRootlessEUID())
logrus.Debugf(" No signature storage configuration found for %s, using built-in default %s", dr.PolicyConfigurationIdentity(), baseURL.Redacted())
}
// NOTE: Keep this in sync with docs/signature-protocols.md!
// FIXME? Restrict to explicitly supported schemes?
@ -181,8 +181,8 @@ func (config *registryConfiguration) lookasideStorageBaseURL(dr dockerReference,
if path.Clean(repo) != repo { // Coverage: This should not be reachable because /./ and /../ components are not valid in docker references
return nil, fmt.Errorf("Unexpected path elements in Docker reference %s for signature storage", dr.ref.String())
}
url.Path = url.Path + "/" + repo
return url, nil
baseURL.Path = baseURL.Path + "/" + repo
return baseURL, nil
}
// builtinDefaultLookasideStorageDir returns default signature storage URL as per euid
@ -201,8 +201,8 @@ func (config *registryConfiguration) signatureTopLevel(ref dockerReference, writ
identity := ref.PolicyConfigurationIdentity()
if ns, ok := config.Docker[identity]; ok {
logrus.Debugf(` Lookaside configuration: using "docker" namespace %s`, identity)
if url := ns.signatureTopLevel(write); url != "" {
return url
if ret := ns.signatureTopLevel(write); ret != "" {
return ret
}
}
@ -210,8 +210,8 @@ func (config *registryConfiguration) signatureTopLevel(ref dockerReference, writ
for _, name := range ref.PolicyConfigurationNamespaces() {
if ns, ok := config.Docker[name]; ok {
logrus.Debugf(` Lookaside configuration: using "docker" namespace %s`, name)
if url := ns.signatureTopLevel(write); url != "" {
return url
if ret := ns.signatureTopLevel(write); ret != "" {
return ret
}
}
}
@ -219,8 +219,8 @@ func (config *registryConfiguration) signatureTopLevel(ref dockerReference, writ
// Look for a default location
if config.DefaultDocker != nil {
logrus.Debugf(` Lookaside configuration: using "default-docker" configuration`)
if url := config.DefaultDocker.signatureTopLevel(write); url != "" {
return url
if ret := config.DefaultDocker.signatureTopLevel(write); ret != "" {
return ret
}
}
return ""
@ -287,7 +287,7 @@ func (ns registryNamespace) signatureTopLevel(write bool) string {
// base is not nil from the caller
// NOTE: Keep this in sync with docs/signature-protocols.md!
func lookasideStorageURL(base lookasideStorageBase, manifestDigest digest.Digest, index int) *url.URL {
url := *base
url.Path = fmt.Sprintf("%s@%s=%s/signature-%d", url.Path, manifestDigest.Algorithm(), manifestDigest.Hex(), index+1)
return &url
sigURL := *base
sigURL.Path = fmt.Sprintf("%s@%s=%s/signature-%d", sigURL.Path, manifestDigest.Algorithm(), manifestDigest.Hex(), index+1)
return &sigURL
}

View File

@ -15,7 +15,7 @@
//
// Second, there are stubs with a constructor, like NoGetBlobAtInitialize. The Initialize marker
// means that a constructor must be called:
//
// type yourSource struct {
// stubs.NoGetBlobAtInitialize
// …

View File

@ -107,7 +107,7 @@ func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest
var err error
if instanceDigest == nil {
dig = digest.Digest(s.descriptor.Digest)
dig = s.descriptor.Digest
mimeType = s.descriptor.MediaType
} else {
dig = *instanceDigest

View File

@ -179,35 +179,25 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) {
return imgspecv1.Descriptor{}, err
}
var d *imgspecv1.Descriptor
if ref.image == "" {
// return manifest if only one image is in the oci directory
if len(index.Manifests) == 1 {
d = &index.Manifests[0]
} else {
if len(index.Manifests) != 1 {
// ask user to choose image when more than one image in the oci directory
return imgspecv1.Descriptor{}, ErrMoreThanOneImage
}
return index.Manifests[0], nil
} else {
// if image specified, look through all manifests for a match
for _, md := range index.Manifests {
if md.MediaType != imgspecv1.MediaTypeImageManifest && md.MediaType != imgspecv1.MediaTypeImageIndex {
continue
}
refName, ok := md.Annotations[imgspecv1.AnnotationRefName]
if !ok {
continue
}
if refName == ref.image {
d = &md
break
if refName, ok := md.Annotations[imgspecv1.AnnotationRefName]; ok && refName == ref.image {
return md, nil
}
}
}
if d == nil {
return imgspecv1.Descriptor{}, ImageNotFoundError{ref}
}
return *d, nil
return imgspecv1.Descriptor{}, ImageNotFoundError{ref}
}
// LoadManifestDescriptor loads the manifest descriptor to be used to retrieve the image name

View File

@ -67,14 +67,14 @@ func newOpenshiftClient(ref openshiftReference) (*openshiftClient, error) {
// doRequest performs a correctly authenticated request to a specified path, and returns response body or an error object.
func (c *openshiftClient) doRequest(ctx context.Context, method, path string, requestBody []byte) ([]byte, error) {
url := *c.baseURL
url.Path = path
requestURL := *c.baseURL
requestURL.Path = path
var requestBodyReader io.Reader
if requestBody != nil {
logrus.Debugf("Will send body: %s", requestBody)
requestBodyReader = bytes.NewReader(requestBody)
}
req, err := http.NewRequestWithContext(ctx, method, url.String(), requestBodyReader)
req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), requestBodyReader)
if err != nil {
return nil, err
}
@ -90,7 +90,7 @@ func (c *openshiftClient) doRequest(ctx context.Context, method, path string, re
req.Header.Set("Content-Type", "application/json")
}
logrus.Debugf("%s %s", method, url.Redacted())
logrus.Debugf("%s %s", method, requestURL.Redacted())
res, err := c.httpClient.Do(req)
if err != nil {
return nil, err

View File

@ -138,7 +138,7 @@ func (s *blobCacheSource) LayerInfosForCopy(ctx context.Context, instanceDigest
var replaceDigest []byte
var err error
blobFile := s.reference.blobPath(info.Digest, false)
alternate := ""
var alternate string
switch s.reference.compress {
case types.Compress:
alternate = blobFile + compressedNote

View File

@ -59,8 +59,6 @@ func parseUnnormalizedShortName(input string) (bool, reference.Named, error) {
// the tag or digest and stores it in the return values so that both can be
// re-added to a possible resolved alias' or USRs at a later point.
func splitUserInput(named reference.Named) (isTagged bool, isDigested bool, normalized reference.Named, tag string, digest digest.Digest) {
normalized = named
tagged, isT := named.(reference.NamedTagged)
if isT {
isTagged = true
@ -170,7 +168,7 @@ func (r *Resolved) Description() string {
// Note that nil is returned if len(pullErrors) == 0. Otherwise, the amount of
// pull errors must equal the amount of pull candidates.
func (r *Resolved) FormatPullErrors(pullErrors []error) error {
if len(pullErrors) >= 0 && len(pullErrors) != len(r.PullCandidates) {
if len(pullErrors) > 0 && len(pullErrors) != len(r.PullCandidates) {
pullErrors = append(pullErrors,
fmt.Errorf("internal error: expected %d instead of %d errors for %d pull candidates",
len(r.PullCandidates), len(pullErrors), len(r.PullCandidates)))

View File

@ -6,9 +6,9 @@ const (
// VersionMajor is for an API incompatible changes
VersionMajor = 5
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 23
VersionMinor = 24
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 1
VersionPatch = 0
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = "-dev"

View File

@ -18,7 +18,6 @@ env:
#### Cache-image names to test with (double-quotes around names are critical)
###
FEDORA_NAME: "fedora-36"
PRIOR_FEDORA_NAME: "fedora-35"
UBUNTU_NAME: "ubuntu-2204"
# GCE project where images live
@ -26,7 +25,6 @@ env:
# VM Image built in containers/automation_images
IMAGE_SUFFIX: "c5878804328480768"
FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
####
@ -90,15 +88,6 @@ fedora_testing_task: &fedora_testing
journal_log_script: '${_JOURNALCMD} || true'
prior_fedora_testing_task:
<<: *fedora_testing
alias: prior_fedora_testing
name: *std_test_name
env:
OS_NAME: "${PRIOR_FEDORA_NAME}"
VM_IMAGE: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
ubuntu_testing_task: &ubuntu_testing
<<: *fedora_testing
alias: ubuntu_testing
@ -117,7 +106,7 @@ lint_task:
env:
CIRRUS_WORKING_DIR: "/go/src/github.com/containers/storage"
container:
image: golang:1.16
image: golang:1.17
modules_cache:
fingerprint_script: cat go.sum
folder: $GOPATH/pkg/mod
@ -140,7 +129,6 @@ meta_task:
# Space-separated list of images used by this repository state
IMGNAMES: |-
${FEDORA_CACHE_IMAGE_NAME}
${PRIOR_FEDORA_CACHE_IMAGE_NAME}
${UBUNTU_CACHE_IMAGE_NAME}
BUILDID: "${CIRRUS_BUILD_ID}"
REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
@ -154,7 +142,7 @@ meta_task:
vendor_task:
container:
image: golang:1.16
image: golang:1.17
modules_cache:
fingerprint_script: cat go.sum
folder: $GOPATH/pkg/mod
@ -167,11 +155,10 @@ success_task:
depends_on:
- lint
- fedora_testing
- prior_fedora_testing
- ubuntu_testing
- meta
- vendor
container:
image: golang:1.16
image: golang:1.17
clone_script: 'mkdir -p "$CIRRUS_WORKING_DIR"' # Source code not needed
script: /bin/true

View File

@ -120,6 +120,6 @@ vendor-in-container:
podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src golang make vendor
vendor:
$(GO) mod tidy
$(GO) mod tidy -compat=1.17
$(GO) mod vendor
$(GO) mod verify

View File

@ -68,11 +68,24 @@ type Container struct {
// rwContainerStore provides bookkeeping for information about Containers.
type rwContainerStore interface {
fileBasedStore
metadataStore
containerBigDataStore
flaggableStore
// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
startWriting() error
// stopWriting releases locks obtained by startWriting.
stopWriting()
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
startReading() error
// stopReading releases locks obtained by startReading.
stopReading()
// Create creates a container that has a specified ID (or generates a
// random one if an empty value is supplied) and optional names,
// based on the specified image, using the specified layer as its
@ -147,12 +160,12 @@ func (c *Container) ProcessLabel() string {
}
func (c *Container) MountOpts() []string {
switch c.Flags[mountOptsFlag].(type) {
switch value := c.Flags[mountOptsFlag].(type) {
case []string:
return c.Flags[mountOptsFlag].([]string)
return value
case []interface{}:
var mountOpts []string
for _, v := range c.Flags[mountOptsFlag].([]interface{}) {
for _, v := range value {
if flag, ok := v.(string); ok {
mountOpts = append(mountOpts, flag)
}
@ -163,6 +176,80 @@ func (c *Container) MountOpts() []string {
}
}
// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
//
// This is an internal implementation detail of containerStore construction, every other caller
// should use startWriting() instead.
func (r *containerStore) startWritingWithReload(canReload bool) error {
r.lockfile.Lock()
succeeded := false
defer func() {
if !succeeded {
r.lockfile.Unlock()
}
}()
if canReload {
if err := r.reloadIfChanged(true); err != nil {
return err
}
}
succeeded = true
return nil
}
// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
func (r *containerStore) startWriting() error {
return r.startWritingWithReload(true)
}
// stopWriting releases locks obtained by startWriting.
func (r *containerStore) stopWriting() {
r.lockfile.Unlock()
}
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
func (r *containerStore) startReading() error {
r.lockfile.RLock()
succeeded := false
defer func() {
if !succeeded {
r.lockfile.Unlock()
}
}()
if err := r.reloadIfChanged(false); err != nil {
return err
}
succeeded = true
return nil
}
// stopReading releases locks obtained by startReading.
func (r *containerStore) stopReading() {
r.lockfile.Unlock()
}
// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *containerStore) reloadIfChanged(lockedForWriting bool) error {
r.loadMut.Lock()
defer r.loadMut.Unlock()
modified, err := r.lockfile.Modified()
if err == nil && modified {
return r.load(lockedForWriting)
}
return err
}
func (r *containerStore) Containers() ([]Container, error) {
containers := make([]Container, len(r.containers))
for i := range r.containers {
@ -183,48 +270,60 @@ func (r *containerStore) datapath(id, key string) string {
return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
}
func (r *containerStore) Load() error {
// load reloads the contents of the store from disk.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *containerStore) load(lockedForWriting bool) error {
needSave := false
rpath := r.containerspath()
data, err := os.ReadFile(rpath)
if err != nil && !os.IsNotExist(err) {
return err
}
containers := []*Container{}
layers := make(map[string]*Container)
idlist := []string{}
ids := make(map[string]*Container)
names := make(map[string]*Container)
if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil {
idlist = make([]string, 0, len(containers))
for n, container := range containers {
idlist = append(idlist, container.ID)
ids[container.ID] = containers[n]
layers[container.LayerID] = containers[n]
for _, name := range container.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
needSave = true
}
names[name] = containers[n]
}
if len(data) != 0 {
if err := json.Unmarshal(data, &containers); err != nil {
return fmt.Errorf("loading %q: %w", rpath, err)
}
}
idlist := make([]string, 0, len(containers))
layers := make(map[string]*Container)
ids := make(map[string]*Container)
names := make(map[string]*Container)
for n, container := range containers {
idlist = append(idlist, container.ID)
ids[container.ID] = containers[n]
layers[container.LayerID] = containers[n]
for _, name := range container.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
needSave = true
}
names[name] = containers[n]
}
}
r.containers = containers
r.idindex = truncindex.NewTruncIndex(idlist) // Invalid values in idlist are ignored: they are not a reason to refuse processing the whole store.
r.byid = ids
r.bylayer = layers
r.byname = names
if needSave {
if !lockedForWriting {
// Eventually, the callers should be modified to retry with a write lock, instead.
return errors.New("container store is inconsistent and the current caller does not hold a write lock")
}
return r.Save()
}
return nil
}
// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
func (r *containerStore) Save() error {
if !r.Locked() {
return errors.New("container store is not locked")
}
r.lockfile.AssertLockedForWriting()
rpath := r.containerspath()
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
@ -236,7 +335,7 @@ func (r *containerStore) Save() error {
if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil {
return err
}
return r.Touch()
return r.lockfile.Touch()
}
func newContainerStore(dir string) (rwContainerStore, error) {
@ -255,9 +354,11 @@ func newContainerStore(dir string) (rwContainerStore, error) {
bylayer: make(map[string]*Container),
byname: make(map[string]*Container),
}
cstore.Lock()
defer cstore.Unlock()
if err := cstore.Load(); err != nil {
if err := cstore.startWritingWithReload(false); err != nil {
return nil, err
}
defer cstore.stopWriting()
if err := cstore.load(true); err != nil {
return nil, err
}
return &cstore, nil
@ -329,33 +430,31 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
if err := hasOverlappingRanges(options.GIDMap); err != nil {
return nil, err
}
if err == nil {
container = &Container{
ID: id,
Names: names,
ImageID: image,
LayerID: layer,
Metadata: metadata,
BigDataNames: []string{},
BigDataSizes: make(map[string]int64),
BigDataDigests: make(map[string]digest.Digest),
Created: time.Now().UTC(),
Flags: copyStringInterfaceMap(options.Flags),
UIDMap: copyIDMap(options.UIDMap),
GIDMap: copyIDMap(options.GIDMap),
}
r.containers = append(r.containers, container)
r.byid[id] = container
// This can only fail on duplicate IDs, which shouldnt happen — and in that case the index is already in the desired state anyway.
// Implementing recovery from an unlikely and unimportant failure here would be too risky.
_ = r.idindex.Add(id)
r.bylayer[layer] = container
for _, name := range names {
r.byname[name] = container
}
err = r.Save()
container = copyContainer(container)
container = &Container{
ID: id,
Names: names,
ImageID: image,
LayerID: layer,
Metadata: metadata,
BigDataNames: []string{},
BigDataSizes: make(map[string]int64),
BigDataDigests: make(map[string]digest.Digest),
Created: time.Now().UTC(),
Flags: copyStringInterfaceMap(options.Flags),
UIDMap: copyIDMap(options.UIDMap),
GIDMap: copyIDMap(options.GIDMap),
}
r.containers = append(r.containers, container)
r.byid[id] = container
// This can only fail on duplicate IDs, which shouldnt happen — and in that case the index is already in the desired state anyway.
// Implementing recovery from an unlikely and unimportant failure here would be too risky.
_ = r.idindex.Add(id)
r.bylayer[layer] = container
for _, name := range names {
r.byname[name] = container
}
err = r.Save()
container = copyContainer(container)
return container, err
}
@ -595,46 +694,3 @@ func (r *containerStore) Wipe() error {
}
return nil
}
func (r *containerStore) Lock() {
r.lockfile.Lock()
}
func (r *containerStore) RLock() {
r.lockfile.RLock()
}
func (r *containerStore) Unlock() {
r.lockfile.Unlock()
}
func (r *containerStore) Touch() error {
return r.lockfile.Touch()
}
func (r *containerStore) Modified() (bool, error) {
return r.lockfile.Modified()
}
func (r *containerStore) IsReadWrite() bool {
return r.lockfile.IsReadWrite()
}
func (r *containerStore) TouchedSince(when time.Time) bool {
return r.lockfile.TouchedSince(when)
}
func (r *containerStore) Locked() bool {
return r.lockfile.Locked()
}
func (r *containerStore) ReloadIfChanged() error {
r.loadMut.Lock()
defer r.loadMut.Unlock()
modified, err := r.Modified()
if err == nil && modified {
return r.Load()
}
return err
}

View File

@ -8,13 +8,12 @@ import (
"path/filepath"
"strings"
"github.com/sirupsen/logrus"
"github.com/vbatts/tar-split/tar/storage"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/directory"
"github.com/containers/storage/pkg/idtools"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
"github.com/vbatts/tar-split/tar/storage"
)
// FsMagic unsigned id of the filesystem in use.

View File

@ -174,7 +174,7 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, options ApplyDiffOpts)
defer driverPut(driver, id, &err)
defaultForceMask := os.FileMode(0700)
var forceMask *os.FileMode = nil
var forceMask *os.FileMode // = nil
if runtime.GOOS == "darwin" {
forceMask = &defaultForceMask
}

View File

@ -8,14 +8,16 @@ import (
"flag"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/containers/storage/pkg/reexec"
"golang.org/x/sys/unix"
)
func init() {
reexec.Register("storage-mountfrom", mountFromMain)
reexec.Register("storage-mountfrom", mountOverlayFromMain)
}
func fatal(err error) {
@ -31,7 +33,7 @@ type mountOptions struct {
Flag uint32
}
func mountFrom(dir, device, target, mType string, flags uintptr, label string) error {
func mountOverlayFrom(dir, device, target, mType string, flags uintptr, label string) error {
options := &mountOptions{
Device: device,
Target: target,
@ -67,7 +69,7 @@ func mountFrom(dir, device, target, mType string, flags uintptr, label string) e
}
// mountfromMain is the entry-point for storage-mountfrom on re-exec.
func mountFromMain() {
func mountOverlayFromMain() {
runtime.LockOSThread()
flag.Parse()
@ -77,11 +79,96 @@ func mountFromMain() {
fatal(err)
}
if err := os.Chdir(flag.Arg(0)); err != nil {
// Mount the arguments passed from the specified directory. Some of the
// paths mentioned in the values we pass to the kernel are relative to
// the specified directory.
homedir := flag.Arg(0)
if err := os.Chdir(homedir); err != nil {
fatal(err)
}
if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil {
pageSize := unix.Getpagesize()
if len(options.Label) < pageSize {
if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil {
fatal(err)
}
os.Exit(0)
}
// Those arguments still took up too much space. Open the diff
// directories and use their descriptor numbers as lowers, using
// /proc/self/fd as the current directory.
// Split out the various options, since we need to manipulate the
// paths, but we don't want to mess with other options.
var upperk, upperv, workk, workv, lowerk, lowerv, labelk, labelv, others string
for _, arg := range strings.Split(options.Label, ",") {
kv := strings.SplitN(arg, "=", 2)
switch kv[0] {
case "upperdir":
upperk = "upperdir="
upperv = kv[1]
case "workdir":
workk = "workdir="
workv = kv[1]
case "lowerdir":
lowerk = "lowerdir="
lowerv = kv[1]
case "label":
labelk = "label="
labelv = kv[1]
default:
if others == "" {
others = arg
} else {
others = others + "," + arg
}
}
}
// Make sure upperdir, workdir, and the target are absolute paths.
if upperv != "" && !filepath.IsAbs(upperv) {
upperv = filepath.Join(homedir, upperv)
}
if workv != "" && !filepath.IsAbs(workv) {
workv = filepath.Join(homedir, workv)
}
if !filepath.IsAbs(options.Target) {
options.Target = filepath.Join(homedir, options.Target)
}
// Get a descriptor for each lower, and use that descriptor's name as
// the new value for the list of lowers, because it's shorter.
if lowerv != "" {
lowers := strings.Split(lowerv, ":")
for i := range lowers {
lowerFd, err := unix.Open(lowers[i], unix.O_RDONLY, 0)
if err != nil {
fatal(err)
}
lowers[i] = fmt.Sprintf("%d", lowerFd)
}
lowerv = strings.Join(lowers, ":")
}
// Reconstruct the Label field.
options.Label = upperk + upperv + "," + workk + workv + "," + lowerk + lowerv + "," + labelk + labelv + "," + others
options.Label = strings.ReplaceAll(options.Label, ",,", ",")
// Okay, try this, if we managed to make the arguments fit.
var err error
if len(options.Label) < pageSize {
if err := os.Chdir("/proc/self/fd"); err != nil {
fatal(err)
}
err = unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label)
} else {
err = fmt.Errorf("cannot mount layer, mount data %q too large %d >= page size %d", options.Label, len(options.Label), pageSize)
}
// Clean up.
if err != nil {
fmt.Fprintf(os.Stderr, "creating overlay mount to %s, mount_data=%q\n", options.Target, options.Label)
fatal(err)
}

View File

@ -73,21 +73,23 @@ const (
// or root directory. Mounts are always done relative to root and
// referencing the symbolic links in order to ensure the number of
// lower directories can fit in a single page for making the mount
// syscall. A hard upper limit of 128 lower layers is enforced to ensure
// syscall. A hard upper limit of 500 lower layers is enforced to ensure
// that mounts do not fail due to length.
const (
linkDir = "l"
lowerFile = "lower"
maxDepth = 128
maxDepth = 500
// idLength represents the number of random characters
// which can be used to create the unique link identifier
// for every layer. If this value is too long then the
// page size limit for the mount command may be exceeded.
// The idLength should be selected such that following equation
// is true (512 is a buffer for label metadata).
// ((idLength + len(linkDir) + 1) * maxDepth) <= (pageSize - 512)
// is true (512 is a buffer for label metadata, 128 is the
// number of lowers we want to be able to use without having
// to use bind mounts to get all the way to the kernel limit).
// ((idLength + len(linkDir) + 1) * 128) <= (pageSize - 512)
idLength = 26
)
@ -1376,27 +1378,9 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
return "", errors.New("max depth exceeded")
}
// absLowers is the list of lowers as absolute paths, which works well with additional stores.
// absLowers is the list of lowers as absolute paths.
absLowers := []string{}
// relLowers is the list of lowers as paths relative to the driver's home directory.
relLowers := []string{}
// Check if $link/../diff{1-*} exist. If they do, add them, in order, as the front of the lowers
// lists that we're building. "diff" itself is the upper, so it won't be in the lists.
link, err := os.ReadFile(path.Join(dir, "link"))
if err != nil {
if !os.IsNotExist(err) {
return "", err
}
logrus.Warnf("Can't read parent link %q because it does not exist. Going through storage to recreate the missing links.", path.Join(dir, "link"))
if err := d.recreateSymlinks(); err != nil {
return "", fmt.Errorf("recreating the links: %w", err)
}
link, err = os.ReadFile(path.Join(dir, "link"))
if err != nil {
return "", err
}
}
diffN := 1
perms := defaultPerms
if d.options.forceMask != nil {
@ -1410,7 +1394,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
}
for err == nil {
absLowers = append(absLowers, filepath.Join(dir, nameWithSuffix("diff", diffN)))
relLowers = append(relLowers, dumbJoin(linkDir, string(link), "..", nameWithSuffix("diff", diffN)))
diffN++
st, err = os.Stat(filepath.Join(dir, nameWithSuffix("diff", diffN)))
if err == nil && !permsKnown {
@ -1435,6 +1418,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
perms = os.FileMode(st2.Mode())
permsKnown = true
}
l = lower
break
}
lower = ""
@ -1459,12 +1443,10 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
lower = newpath
}
absLowers = append(absLowers, lower)
relLowers = append(relLowers, l)
diffN = 1
_, err = os.Stat(dumbJoin(lower, "..", nameWithSuffix("diff", diffN)))
for err == nil {
absLowers = append(absLowers, dumbJoin(lower, "..", nameWithSuffix("diff", diffN)))
relLowers = append(relLowers, dumbJoin(l, "..", nameWithSuffix("diff", diffN)))
diffN++
_, err = os.Stat(dumbJoin(lower, "..", nameWithSuffix("diff", diffN)))
}
@ -1472,7 +1454,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
if len(absLowers) == 0 {
absLowers = append(absLowers, path.Join(dir, "empty"))
relLowers = append(relLowers, path.Join(id, "empty"))
}
// user namespace requires this to move a directory from lower to upper.
rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
@ -1606,28 +1587,23 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
return nil
}
} else if len(mountData) >= pageSize {
// Use relative paths and mountFrom when the mount data has exceeded
// the page size. The mount syscall fails if the mount data cannot
// fit within a page and relative links make the mount data much
// smaller at the expense of requiring a fork exec to chroot.
// Use mountFrom when the mount data has exceeded the page size. The mount syscall fails if
// the mount data cannot fit within a page and relative links make the mount data much
// smaller at the expense of requiring a fork exec to chdir().
workdir = path.Join(id, "work")
//FIXME: We need to figure out to get this to work with additional stores
if readWrite {
diffDir := path.Join(id, "diff")
opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), diffDir, workdir)
opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, workdir)
} else {
opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(relLowers, ":"))
opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(absLowers, ":"))
}
if len(optsList) > 0 {
opts = fmt.Sprintf("%s,%s", opts, strings.Join(optsList, ","))
}
mountData = label.FormatMountLabel(opts, options.MountLabel)
if len(mountData) >= pageSize {
return "", fmt.Errorf("cannot mount layer, mount label %q too large %d >= page size %d", options.MountLabel, len(mountData), pageSize)
}
mountFunc = func(source string, target string, mType string, flags uintptr, label string) error {
return mountFrom(d.home, source, target, mType, flags, label)
return mountOverlayFrom(d.home, source, target, mType, flags, label)
}
mountTarget = path.Join(id, "merged")
}

View File

@ -1,9 +1,8 @@
package graphdriver
import (
"github.com/sirupsen/logrus"
"github.com/containers/storage/pkg/idtools"
"github.com/sirupsen/logrus"
)
// TemplateDriver is just barely enough of a driver that we can implement a

View File

@ -194,7 +194,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool
if parent != "" {
parentDir, err := d.Get(parent, graphdriver.MountOpts{})
if err != nil {
return fmt.Errorf("%s: %s", parent, err)
return fmt.Errorf("%s: %w", parent, err)
}
if err := dirCopy(parentDir, dir); err != nil {
return err

View File

@ -1,11 +1,4 @@
//go:build !linux && !freebsd
// +build !linux,!freebsd
package zfs
func checkRootdirFs(rootdir string) error {
return nil
}
func getMountpoint(id string) string {
return id
}

View File

@ -1,7 +1,6 @@
package storage
import (
"errors"
"fmt"
"os"
"path/filepath"
@ -96,10 +95,16 @@ type Image struct {
// roImageStore provides bookkeeping for information about Images.
type roImageStore interface {
roFileBasedStore
roMetadataStore
roBigDataStore
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
startReading() error
// stopReading releases locks obtained by startReading.
stopReading()
// Exists checks if there is an image with the given ID or name.
Exists(id string) bool
@ -119,11 +124,17 @@ type roImageStore interface {
// rwImageStore provides bookkeeping for information about Images.
type rwImageStore interface {
roImageStore
rwFileBasedStore
rwMetadataStore
rwImageBigDataStore
flaggableStore
// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
startWriting() error
// stopWriting releases locks obtained by startWriting.
stopWriting()
// Create creates an image that has a specified ID (or a random one) and
// optional names, using the specified layer as its topmost (hopefully
// read-only) layer. That layer can be referenced by multiple images.
@ -142,7 +153,7 @@ type rwImageStore interface {
}
type imageStore struct {
lockfile Locker
lockfile Locker // lockfile.IsReadWrite can be used to distinguish between read-write and read-only image stores.
dir string
images []*Image
idindex *truncindex.TruncIndex
@ -182,6 +193,91 @@ func copyImageSlice(slice []*Image) []*Image {
return nil
}
// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
//
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startWritingWithReload(canReload bool) error {
r.lockfile.Lock()
succeeded := false
defer func() {
if !succeeded {
r.lockfile.Unlock()
}
}()
if canReload {
if err := r.reloadIfChanged(true); err != nil {
return err
}
}
succeeded = true
return nil
}
// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
func (r *imageStore) startWriting() error {
return r.startWritingWithReload(true)
}
// stopWriting releases locks obtained by startWriting.
func (r *imageStore) stopWriting() {
r.lockfile.Unlock()
}
// startReadingWithReload makes sure the store is fresh if canReload, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
//
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startReadingWithReload(canReload bool) error {
r.lockfile.RLock()
succeeded := false
defer func() {
if !succeeded {
r.lockfile.Unlock()
}
}()
if canReload {
if err := r.reloadIfChanged(false); err != nil {
return err
}
}
succeeded = true
return nil
}
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
func (r *imageStore) startReading() error {
return r.startReadingWithReload(true)
}
// stopReading releases locks obtained by startReading.
func (r *imageStore) stopReading() {
r.lockfile.Unlock()
}
// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *imageStore) reloadIfChanged(lockedForWriting bool) error {
r.loadMut.Lock()
defer r.loadMut.Unlock()
modified, err := r.lockfile.Modified()
if err == nil && modified {
return r.load(lockedForWriting)
}
return err
}
func (r *imageStore) Images() ([]Image, error) {
images := make([]Image, len(r.images))
for i := range r.images {
@ -242,45 +338,53 @@ func (i *Image) recomputeDigests() error {
return nil
}
func (r *imageStore) Load() error {
// load reloads the contents of the store from disk.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *imageStore) load(lockedForWriting bool) error {
shouldSave := false
rpath := r.imagespath()
data, err := os.ReadFile(rpath)
if err != nil && !os.IsNotExist(err) {
return err
}
images := []*Image{}
idlist := []string{}
if len(data) != 0 {
if err := json.Unmarshal(data, &images); err != nil {
return fmt.Errorf("loading %q: %w", rpath, err)
}
}
idlist := make([]string, 0, len(images))
ids := make(map[string]*Image)
names := make(map[string]*Image)
digests := make(map[digest.Digest][]*Image)
if err = json.Unmarshal(data, &images); len(data) == 0 || err == nil {
idlist = make([]string, 0, len(images))
for n, image := range images {
ids[image.ID] = images[n]
idlist = append(idlist, image.ID)
for _, name := range image.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
shouldSave = true
}
for n, image := range images {
ids[image.ID] = images[n]
idlist = append(idlist, image.ID)
for _, name := range image.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
shouldSave = true
}
// Compute the digest list.
err = image.recomputeDigests()
if err != nil {
return fmt.Errorf("computing digests for image with ID %q (%v): %w", image.ID, image.Names, err)
}
for _, name := range image.Names {
names[name] = image
}
for _, digest := range image.Digests {
list := digests[digest]
digests[digest] = append(list, image)
}
image.ReadOnly = !r.IsReadWrite()
}
// Compute the digest list.
if err := image.recomputeDigests(); err != nil {
return fmt.Errorf("computing digests for image with ID %q (%v): %w", image.ID, image.Names, err)
}
for _, name := range image.Names {
names[name] = image
}
for _, digest := range image.Digests {
list := digests[digest]
digests[digest] = append(list, image)
}
image.ReadOnly = !r.lockfile.IsReadWrite()
}
if shouldSave && (!r.IsReadWrite() || !r.Locked()) {
if shouldSave && (!r.lockfile.IsReadWrite() || !lockedForWriting) {
// Eventually, the callers should be modified to retry with a write lock if IsReadWrite && !lockedForWriting, instead.
return ErrDuplicateImageNames
}
r.images = images
@ -294,13 +398,13 @@ func (r *imageStore) Load() error {
return nil
}
// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
func (r *imageStore) Save() error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify the image store at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
if !r.Locked() {
return errors.New("image store is not locked for writing")
}
r.lockfile.AssertLockedForWriting()
rpath := r.imagespath()
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
@ -312,7 +416,7 @@ func (r *imageStore) Save() error {
if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil {
return err
}
return r.Touch()
return r.lockfile.Touch()
}
func newImageStore(dir string) (rwImageStore, error) {
@ -331,9 +435,11 @@ func newImageStore(dir string) (rwImageStore, error) {
byname: make(map[string]*Image),
bydigest: make(map[digest.Digest][]*Image),
}
istore.Lock()
defer istore.Unlock()
if err := istore.Load(); err != nil {
if err := istore.startWritingWithReload(false); err != nil {
return nil, err
}
defer istore.stopWriting()
if err := istore.load(true); err != nil {
return nil, err
}
return &istore, nil
@ -352,9 +458,11 @@ func newROImageStore(dir string) (roImageStore, error) {
byname: make(map[string]*Image),
bydigest: make(map[digest.Digest][]*Image),
}
istore.RLock()
defer istore.Unlock()
if err := istore.Load(); err != nil {
if err := istore.startReadingWithReload(false); err != nil {
return nil, err
}
defer istore.stopReading()
if err := istore.load(false); err != nil {
return nil, err
}
return &istore, nil
@ -373,7 +481,7 @@ func (r *imageStore) lookup(id string) (*Image, bool) {
}
func (r *imageStore) ClearFlag(id string, flag string) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to clear flags on images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
image, ok := r.lookup(id)
@ -385,7 +493,7 @@ func (r *imageStore) ClearFlag(id string, flag string) error {
}
func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to set flags on images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
image, ok := r.lookup(id)
@ -400,7 +508,7 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
}
func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (image *Image, err error) {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return nil, fmt.Errorf("not allowed to create new images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
if id == "" {
@ -487,7 +595,7 @@ func (r *imageStore) Metadata(id string) (string, error) {
}
func (r *imageStore) SetMetadata(id, metadata string) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify image metadata at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
if image, ok := r.lookup(id); ok {
@ -506,7 +614,7 @@ func (i *Image) addNameToHistory(name string) {
}
func (r *imageStore) updateNames(id string, names []string, op updateNameOperation) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to change image name assignments at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
image, ok := r.lookup(id)
@ -533,7 +641,7 @@ func (r *imageStore) updateNames(id string, names []string, op updateNameOperati
}
func (r *imageStore) Delete(id string) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
image, ok := r.lookup(id)
@ -669,7 +777,7 @@ func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func
if key == "" {
return fmt.Errorf("can't set empty name for image big data item: %w", ErrInvalidBigDataName)
}
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to save data items associated with images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
image, ok := r.lookup(id)
@ -750,7 +858,7 @@ func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func
}
func (r *imageStore) Wipe() error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
}
ids := make([]string, 0, len(r.byid))
@ -764,46 +872,3 @@ func (r *imageStore) Wipe() error {
}
return nil
}
func (r *imageStore) Lock() {
r.lockfile.Lock()
}
func (r *imageStore) RLock() {
r.lockfile.RLock()
}
func (r *imageStore) Unlock() {
r.lockfile.Unlock()
}
func (r *imageStore) Touch() error {
return r.lockfile.Touch()
}
func (r *imageStore) Modified() (bool, error) {
return r.lockfile.Modified()
}
func (r *imageStore) IsReadWrite() bool {
return r.lockfile.IsReadWrite()
}
func (r *imageStore) TouchedSince(when time.Time) bool {
return r.lockfile.TouchedSince(when)
}
func (r *imageStore) Locked() bool {
return r.lockfile.Locked()
}
func (r *imageStore) ReloadIfChanged() error {
r.loadMut.Lock()
defer r.loadMut.Unlock()
modified, err := r.Modified()
if err == nil && modified {
return r.Load()
}
return err
}

View File

@ -141,10 +141,16 @@ type DiffOptions struct {
// name, and keeping track of parent-child relationships, along with a list of
// all known layers.
type roLayerStore interface {
roFileBasedStore
roMetadataStore
roLayerBigDataStore
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
startReading() error
// stopReading releases locks obtained by startReading.
stopReading()
// Exists checks if a layer with the specified name or ID is known.
Exists(id string) bool
@ -194,11 +200,17 @@ type roLayerStore interface {
// all known layers.
type rwLayerStore interface {
roLayerStore
rwFileBasedStore
rwMetadataStore
flaggableStore
rwLayerBigDataStore
// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
startWriting() error
// stopWriting releases locks obtained by startWriting.
stopWriting()
// Create creates a new layer, optionally giving it a specified ID rather than
// a randomly-generated one, either inheriting data from another specified
// layer or the empty base layer. The new layer can optionally be given names
@ -304,6 +316,125 @@ func copyLayer(l *Layer) *Layer {
}
}
// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
//
// This is an internal implementation detail of layerStore construction, every other caller
// should use startWriting() instead.
func (r *layerStore) startWritingWithReload(canReload bool) error {
r.lockfile.Lock()
succeeded := false
defer func() {
if !succeeded {
r.lockfile.Unlock()
}
}()
if canReload {
if err := r.reloadIfChanged(true); err != nil {
return err
}
}
succeeded = true
return nil
}
// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
func (r *layerStore) startWriting() error {
return r.startWritingWithReload(true)
}
// stopWriting releases locks obtained by startWriting.
func (r *layerStore) stopWriting() {
r.lockfile.Unlock()
}
// startReadingWithReload makes sure the store is fresh if canReload, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
//
// This is an internal implementation detail of layerStore construction, every other caller
// should use startReading() instead.
func (r *layerStore) startReadingWithReload(canReload bool) error {
r.lockfile.RLock()
succeeded := false
defer func() {
if !succeeded {
r.lockfile.Unlock()
}
}()
if canReload {
if err := r.reloadIfChanged(false); err != nil {
return err
}
}
succeeded = true
return nil
}
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
func (r *layerStore) startReading() error {
return r.startReadingWithReload(true)
}
// stopReading releases locks obtained by startReading.
func (r *layerStore) stopReading() {
r.lockfile.Unlock()
}
// Modified() checks if the most recent writer was a party other than the
// last recorded writer. It should only be called with the lock held.
func (r *layerStore) Modified() (bool, error) {
var mmodified, tmodified bool
lmodified, err := r.lockfile.Modified()
if err != nil {
return lmodified, err
}
if r.lockfile.IsReadWrite() {
r.mountsLockfile.RLock()
defer r.mountsLockfile.Unlock()
mmodified, err = r.mountsLockfile.Modified()
if err != nil {
return lmodified, err
}
}
if lmodified || mmodified {
return true, nil
}
// If the layers.json file has been modified manually, then we have to
// reload the storage in any case.
info, err := os.Stat(r.layerspath())
if err != nil && !os.IsNotExist(err) {
return false, fmt.Errorf("stat layers file: %w", err)
}
if info != nil {
tmodified = info.ModTime() != r.layerspathModified
}
return tmodified, nil
}
// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *layerStore) reloadIfChanged(lockedForWriting bool) error {
r.loadMut.Lock()
defer r.loadMut.Unlock()
modified, err := r.Modified()
if err == nil && modified {
return r.load(lockedForWriting)
}
return err
}
func (r *layerStore) Layers() ([]Layer, error) {
layers := make([]Layer, len(r.layers))
for i := range r.layers {
@ -320,7 +451,11 @@ func (r *layerStore) layerspath() string {
return filepath.Join(r.layerdir, "layers.json")
}
func (r *layerStore) Load() error {
// load reloads the contents of the store from disk.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *layerStore) load(lockedForWriting bool) error {
shouldSave := false
rpath := r.layerspath()
info, err := os.Stat(rpath)
@ -335,41 +470,45 @@ func (r *layerStore) Load() error {
if err != nil && !os.IsNotExist(err) {
return err
}
layers := []*Layer{}
idlist := []string{}
if len(data) != 0 {
if err := json.Unmarshal(data, &layers); err != nil {
return fmt.Errorf("loading %q: %w", rpath, err)
}
}
idlist := make([]string, 0, len(layers))
ids := make(map[string]*Layer)
names := make(map[string]*Layer)
compressedsums := make(map[digest.Digest][]string)
uncompressedsums := make(map[digest.Digest][]string)
if r.IsReadWrite() {
if r.lockfile.IsReadWrite() {
selinux.ClearLabels()
}
if err = json.Unmarshal(data, &layers); len(data) == 0 || err == nil {
idlist = make([]string, 0, len(layers))
for n, layer := range layers {
ids[layer.ID] = layers[n]
idlist = append(idlist, layer.ID)
for _, name := range layer.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
shouldSave = true
}
names[name] = layers[n]
for n, layer := range layers {
ids[layer.ID] = layers[n]
idlist = append(idlist, layer.ID)
for _, name := range layer.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
shouldSave = true
}
if layer.CompressedDigest != "" {
compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID)
}
if layer.UncompressedDigest != "" {
uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID)
}
if layer.MountLabel != "" {
selinux.ReserveLabel(layer.MountLabel)
}
layer.ReadOnly = !r.IsReadWrite()
names[name] = layers[n]
}
err = nil
if layer.CompressedDigest != "" {
compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID)
}
if layer.UncompressedDigest != "" {
uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID)
}
if layer.MountLabel != "" {
selinux.ReserveLabel(layer.MountLabel)
}
layer.ReadOnly = !r.lockfile.IsReadWrite()
}
if shouldSave && (!r.IsReadWrite() || !r.Locked()) {
if shouldSave && (!r.lockfile.IsReadWrite() || !lockedForWriting) {
// Eventually, the callers should be modified to retry with a write lock if IsReadWrite && !lockedForWriting, instead.
return ErrDuplicateLayerNames
}
r.layers = layers
@ -380,17 +519,18 @@ func (r *layerStore) Load() error {
r.byuncompressedsum = uncompressedsums
// Load and merge information about which layers are mounted, and where.
if r.IsReadWrite() {
if r.lockfile.IsReadWrite() {
r.mountsLockfile.RLock()
defer r.mountsLockfile.Unlock()
if err = r.loadMounts(); err != nil {
if err := r.loadMounts(); err != nil {
return err
}
// Last step: as were writable, try to remove anything that a previous
// user of this storage area marked for deletion but didn't manage to
// actually delete.
if r.Locked() {
var incompleteDeletionErrors error // = nil
if lockedForWriting {
for _, layer := range r.layers {
if layer.Flags == nil {
layer.Flags = make(map[string]interface{})
@ -399,18 +539,26 @@ func (r *layerStore) Load() error {
logrus.Warnf("Found incomplete layer %#v, deleting it", layer.ID)
err = r.deleteInternal(layer.ID)
if err != nil {
break
// Don't return the error immediately, because deleteInternal does not saveLayers();
// Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully
// deleted incomplete layers have their metadata correctly removed.
incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors,
fmt.Errorf("deleting layer %#v: %w", layer.ID, err))
}
shouldSave = true
}
}
}
if shouldSave {
return r.saveLayers()
if err := r.saveLayers(); err != nil {
return err
}
}
if incompleteDeletionErrors != nil {
return incompleteDeletionErrors
}
}
return err
return nil
}
func (r *layerStore) loadMounts() error {
@ -450,6 +598,8 @@ func (r *layerStore) loadMounts() error {
return err
}
// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
func (r *layerStore) Save() error {
r.mountsLockfile.Lock()
defer r.mountsLockfile.Unlock()
@ -460,12 +610,10 @@ func (r *layerStore) Save() error {
}
func (r *layerStore) saveLayers() error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
if !r.Locked() {
return errors.New("layer store is not locked for writing")
}
r.lockfile.AssertLockedForWriting()
rpath := r.layerspath()
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
@ -477,16 +625,14 @@ func (r *layerStore) saveLayers() error {
if err := ioutils.AtomicWriteFile(rpath, jldata, 0600); err != nil {
return err
}
return r.Touch()
return r.lockfile.Touch()
}
func (r *layerStore) saveMounts() error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
if !r.mountsLockfile.Locked() {
return errors.New("layer store mount information is not locked for writing")
}
r.mountsLockfile.AssertLockedForWriting()
mpath := r.mountspath()
if err := os.MkdirAll(filepath.Dir(mpath), 0700); err != nil {
return err
@ -539,9 +685,11 @@ func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Dri
bymount: make(map[string]*Layer),
byname: make(map[string]*Layer),
}
rlstore.Lock()
defer rlstore.Unlock()
if err := rlstore.Load(); err != nil {
if err := rlstore.startWritingWithReload(false); err != nil {
return nil, err
}
defer rlstore.stopWriting()
if err := rlstore.load(true); err != nil {
return nil, err
}
return &rlstore, nil
@ -562,9 +710,11 @@ func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (roL
bymount: make(map[string]*Layer),
byname: make(map[string]*Layer),
}
rlstore.RLock()
defer rlstore.Unlock()
if err := rlstore.Load(); err != nil {
if err := rlstore.startReadingWithReload(false); err != nil {
return nil, err
}
defer rlstore.stopReading()
if err := rlstore.load(false); err != nil {
return nil, err
}
return &rlstore, nil
@ -597,7 +747,7 @@ func (r *layerStore) Size(name string) (int64, error) {
}
func (r *layerStore) ClearFlag(id string, flag string) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to clear flags on layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
layer, ok := r.lookup(id)
@ -609,7 +759,7 @@ func (r *layerStore) ClearFlag(id string, flag string) error {
}
func (r *layerStore) SetFlag(id string, flag string, value interface{}) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to set flags on layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
layer, ok := r.lookup(id)
@ -684,7 +834,7 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s
}
func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return nil, -1, fmt.Errorf("not allowed to create new layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
if err := os.MkdirAll(r.rundir, 0700); err != nil {
@ -889,7 +1039,7 @@ func (r *layerStore) Create(id string, parent *Layer, names []string, mountLabel
}
func (r *layerStore) Mounted(id string) (int, error) {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return 0, fmt.Errorf("no mount information for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
r.mountsLockfile.RLock()
@ -919,7 +1069,7 @@ func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error)
// You are not allowed to mount layers from readonly stores if they
// are not mounted read/only.
if !r.IsReadWrite() && !hasReadOnlyOpt(options.Options) {
if !r.lockfile.IsReadWrite() && !hasReadOnlyOpt(options.Options) {
return "", fmt.Errorf("not allowed to update mount locations for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
r.mountsLockfile.Lock()
@ -969,7 +1119,7 @@ func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error)
}
func (r *layerStore) Unmount(id string, force bool) (bool, error) {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return false, fmt.Errorf("not allowed to update mount locations for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
r.mountsLockfile.Lock()
@ -1007,7 +1157,7 @@ func (r *layerStore) Unmount(id string, force bool) (bool, error) {
}
func (r *layerStore) ParentOwners(id string) (uids, gids []int, err error) {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return nil, nil, fmt.Errorf("no mount information for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
r.mountsLockfile.RLock()
@ -1082,7 +1232,7 @@ func (r *layerStore) removeName(layer *Layer, name string) {
}
func (r *layerStore) updateNames(id string, names []string, op updateNameOperation) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to change layer name assignments at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
layer, ok := r.lookup(id)
@ -1130,7 +1280,7 @@ func (r *layerStore) SetBigData(id, key string, data io.Reader) error {
if key == "" {
return fmt.Errorf("can't set empty name for layer big data item: %w", ErrInvalidBigDataName)
}
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to save data items associated with layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
layer, ok := r.lookup(id)
@ -1189,7 +1339,7 @@ func (r *layerStore) Metadata(id string) (string, error) {
}
func (r *layerStore) SetMetadata(id, metadata string) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify layer metadata at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
if layer, ok := r.lookup(id); ok {
@ -1215,7 +1365,7 @@ func layerHasIncompleteFlag(layer *Layer) bool {
}
func (r *layerStore) deleteInternal(id string) error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
layer, ok := r.lookup(id)
@ -1235,8 +1385,7 @@ func (r *layerStore) deleteInternal(id string) error {
// We never unset incompleteFlag; below, we remove the entire object from r.layers.
id = layer.ID
err := r.driver.Remove(id)
if err != nil {
if err := r.driver.Remove(id); err != nil {
return err
}
@ -1347,7 +1496,7 @@ func (r *layerStore) Get(id string) (*Layer, error) {
}
func (r *layerStore) Wipe() error {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
ids := make([]string, 0, len(r.byid))
@ -1612,7 +1761,7 @@ func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error
}
func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (size int64, err error) {
if !r.IsReadWrite() {
if !r.lockfile.IsReadWrite() {
return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerspath(), ErrStoreIsReadOnly)
}
@ -1699,13 +1848,11 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions,
return -1, err
}
compressor.Close()
if err == nil {
if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil {
return -1, err
}
if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0600); err != nil {
return -1, err
}
if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil {
return -1, err
}
if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0600); err != nil {
return -1, err
}
if compressedDigester != nil {
compressedDigest = compressedDigester.Digest()
@ -1868,77 +2015,6 @@ func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error
return r.layersByDigestMap(r.byuncompressedsum, d)
}
func (r *layerStore) Lock() {
r.lockfile.Lock()
}
func (r *layerStore) RLock() {
r.lockfile.RLock()
}
func (r *layerStore) Unlock() {
r.lockfile.Unlock()
}
func (r *layerStore) Touch() error {
return r.lockfile.Touch()
}
func (r *layerStore) Modified() (bool, error) {
var mmodified, tmodified bool
lmodified, err := r.lockfile.Modified()
if err != nil {
return lmodified, err
}
if r.IsReadWrite() {
r.mountsLockfile.RLock()
defer r.mountsLockfile.Unlock()
mmodified, err = r.mountsLockfile.Modified()
if err != nil {
return lmodified, err
}
}
if lmodified || mmodified {
return true, nil
}
// If the layers.json file has been modified manually, then we have to
// reload the storage in any case.
info, err := os.Stat(r.layerspath())
if err != nil && !os.IsNotExist(err) {
return false, fmt.Errorf("stat layers file: %w", err)
}
if info != nil {
tmodified = info.ModTime() != r.layerspathModified
}
return tmodified, nil
}
func (r *layerStore) IsReadWrite() bool {
return r.lockfile.IsReadWrite()
}
func (r *layerStore) TouchedSince(when time.Time) bool {
return r.lockfile.TouchedSince(when)
}
func (r *layerStore) Locked() bool {
return r.lockfile.Locked()
}
func (r *layerStore) ReloadIfChanged() error {
r.loadMut.Lock()
defer r.loadMut.Unlock()
modified, err := r.Modified()
if err == nil && modified {
return r.Load()
}
return err
}
func closeAll(closes ...func() error) (rErr error) {
for _, f := range closes {
if err := f(); err != nil {

View File

@ -527,6 +527,9 @@ func (ta *tarAppender) addTarFile(path, name string) error {
if err := ReadUserXattrToTarHeader(path, hdr); err != nil {
return err
}
if err := ReadFileFlagsToTarHeader(path, hdr); err != nil {
return err
}
if ta.CopyPass {
copyPassHeader(hdr)
}
@ -770,6 +773,15 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
}
// We defer setting flags on directories until the end of
// Unpack or UnpackLayer in case setting them makes the
// directory immutable.
if hdr.Typeflag != tar.TypeDir {
if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil {
return err
}
}
if len(errs) > 0 {
logrus.WithFields(logrus.Fields{
"errors": errs,
@ -864,7 +876,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
rebaseName := options.RebaseNames[include]
walkRoot := getWalkRoot(srcPath, include)
filepath.WalkDir(walkRoot, func(filePath string, d fs.DirEntry, err error) error {
if err := filepath.WalkDir(walkRoot, func(filePath string, d fs.DirEntry, err error) error {
if err != nil {
logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)
return nil
@ -891,8 +903,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
if include != relFilePath {
matches, err := pm.IsMatch(relFilePath)
if err != nil {
logrus.Errorf("Matching %s: %v", relFilePath, err)
return err
return fmt.Errorf("matching %s: %w", relFilePath, err)
}
skip = matches
}
@ -955,7 +966,10 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
}
}
return nil
})
}); err != nil {
logrus.Errorf("%s", err)
return
}
}
}()
@ -1099,6 +1113,9 @@ loop:
if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
return err
}
if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil {
return err
}
}
return nil
}

View File

@ -50,8 +50,8 @@ func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (
// Currently go does not fill in the major/minors
if s.Mode&unix.S_IFBLK != 0 ||
s.Mode&unix.S_IFCHR != 0 {
hdr.Devmajor = int64(major(uint64(s.Rdev))) // nolint: unconvert
hdr.Devminor = int64(minor(uint64(s.Rdev))) // nolint: unconvert
hdr.Devmajor = int64(major(uint64(s.Rdev))) //nolint: unconvert
hdr.Devminor = int64(minor(uint64(s.Rdev))) //nolint: unconvert
}
}

View File

@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package archive
@ -29,6 +30,7 @@ func statDifferent(oldStat *system.StatT, oldInfo *FileInfo, newStat *system.Sta
if oldStat.Mode() != newStat.Mode() ||
ownerChanged ||
oldStat.Rdev() != newStat.Rdev() ||
oldStat.Flags() != newStat.Flags() ||
// Don't look at size for dirs, its not a good measure of change
(oldStat.Mode()&unix.S_IFDIR != unix.S_IFDIR &&
(!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) {

View File

@ -145,6 +145,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
return nil
}
if _, exists := unpackedPaths[path]; !exists {
if err := resetImmutable(path, nil); err != nil {
return err
}
err := os.RemoveAll(path)
return err
}
@ -156,6 +159,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
} else {
originalBase := base[len(WhiteoutPrefix):]
originalPath := filepath.Join(dir, originalBase)
if err := resetImmutable(originalPath, nil); err != nil {
return 0, err
}
if err := os.RemoveAll(originalPath); err != nil {
return 0, err
}
@ -165,7 +171,15 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
// The only exception is when it is a directory *and* the file from
// the layer is also a directory. Then we want to merge them (i.e.
// just apply the metadata from the layer).
//
// We always reset the immutable flag (if present) to allow metadata
// changes and to allow directory modification. The flag will be
// re-applied based on the contents of hdr either at the end for
// directories or in createTarFile otherwise.
if fi, err := os.Lstat(path); err == nil {
if err := resetImmutable(path, &fi); err != nil {
return 0, err
}
if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
if err := os.RemoveAll(path); err != nil {
return 0, err
@ -215,6 +229,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
return 0, err
}
if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil {
return 0, err
}
}
return size, nil

View File

@ -0,0 +1,167 @@
//go:build freebsd
// +build freebsd
package archive
import (
"archive/tar"
"fmt"
"math/bits"
"os"
"strings"
"syscall"
"github.com/containers/storage/pkg/system"
)
const (
paxSCHILYFflags = "SCHILY.fflags"
)
var (
flagNameToValue = map[string]uint32{
"sappnd": system.SF_APPEND,
"sappend": system.SF_APPEND,
"arch": system.SF_ARCHIVED,
"archived": system.SF_ARCHIVED,
"schg": system.SF_IMMUTABLE,
"schange": system.SF_IMMUTABLE,
"simmutable": system.SF_IMMUTABLE,
"sunlnk": system.SF_NOUNLINK,
"sunlink": system.SF_NOUNLINK,
"snapshot": system.SF_SNAPSHOT,
"uappnd": system.UF_APPEND,
"uappend": system.UF_APPEND,
"uarch": system.UF_ARCHIVE,
"uarchive": system.UF_ARCHIVE,
"hidden": system.UF_HIDDEN,
"uhidden": system.UF_HIDDEN,
"uchg": system.UF_IMMUTABLE,
"uchange": system.UF_IMMUTABLE,
"uimmutable": system.UF_IMMUTABLE,
"uunlnk": system.UF_NOUNLINK,
"uunlink": system.UF_NOUNLINK,
"offline": system.UF_OFFLINE,
"uoffline": system.UF_OFFLINE,
"opaque": system.UF_OPAQUE,
"rdonly": system.UF_READONLY,
"urdonly": system.UF_READONLY,
"readonly": system.UF_READONLY,
"ureadonly": system.UF_READONLY,
"reparse": system.UF_REPARSE,
"ureparse": system.UF_REPARSE,
"sparse": system.UF_SPARSE,
"usparse": system.UF_SPARSE,
"system": system.UF_SYSTEM,
"usystem": system.UF_SYSTEM,
}
// Only include the short names for the reverse map
flagValueToName = map[uint32]string{
system.SF_APPEND: "sappnd",
system.SF_ARCHIVED: "arch",
system.SF_IMMUTABLE: "schg",
system.SF_NOUNLINK: "sunlnk",
system.SF_SNAPSHOT: "snapshot",
system.UF_APPEND: "uappnd",
system.UF_ARCHIVE: "uarch",
system.UF_HIDDEN: "hidden",
system.UF_IMMUTABLE: "uchg",
system.UF_NOUNLINK: "uunlnk",
system.UF_OFFLINE: "offline",
system.UF_OPAQUE: "opaque",
system.UF_READONLY: "rdonly",
system.UF_REPARSE: "reparse",
system.UF_SPARSE: "sparse",
system.UF_SYSTEM: "system",
}
)
func parseFileFlags(fflags string) (uint32, uint32, error) {
var set, clear uint32 = 0, 0
for _, fflag := range strings.Split(fflags, ",") {
isClear := false
if strings.HasPrefix(fflag, "no") {
isClear = true
fflag = strings.TrimPrefix(fflag, "no")
}
if value, ok := flagNameToValue[fflag]; ok {
if isClear {
clear |= value
} else {
set |= value
}
} else {
return 0, 0, fmt.Errorf("parsing file flags, unrecognised token: %s", fflag)
}
}
return set, clear, nil
}
func formatFileFlags(fflags uint32) (string, error) {
var res = []string{}
for fflags != 0 {
// Extract lowest set bit
fflag := uint32(1) << bits.TrailingZeros32(fflags)
if name, ok := flagValueToName[fflag]; ok {
res = append(res, name)
} else {
return "", fmt.Errorf("formatting file flags, unrecognised flag: %x", fflag)
}
fflags &= ^fflag
}
return strings.Join(res, ","), nil
}
func ReadFileFlagsToTarHeader(path string, hdr *tar.Header) error {
st, err := system.Lstat(path)
if err != nil {
return err
}
fflags, err := formatFileFlags(st.Flags())
if err != nil {
return err
}
if fflags != "" {
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[paxSCHILYFflags] = fflags
}
return nil
}
func WriteFileFlagsFromTarHeader(path string, hdr *tar.Header) error {
if fflags, ok := hdr.PAXRecords[paxSCHILYFflags]; ok {
var set, clear uint32
set, clear, err := parseFileFlags(fflags)
if err != nil {
return err
}
// Apply the delta to the existing file flags
st, err := system.Lstat(path)
if err != nil {
return err
}
return system.Lchflags(path, (st.Flags() & ^clear)|set)
}
return nil
}
func resetImmutable(path string, fi *os.FileInfo) error {
var flags uint32
if fi != nil {
flags = (*fi).Sys().(*syscall.Stat_t).Flags
} else {
st, err := system.Lstat(path)
if err != nil {
return err
}
flags = st.Flags()
}
if flags&(system.SF_IMMUTABLE|system.UF_IMMUTABLE) != 0 {
flags &= ^(system.SF_IMMUTABLE | system.UF_IMMUTABLE)
return system.Lchflags(path, flags)
}
return nil
}

View File

@ -0,0 +1,21 @@
//go:build !freebsd
// +build !freebsd
package archive
import (
"archive/tar"
"os"
)
func ReadFileFlagsToTarHeader(path string, hdr *tar.Header) error {
return nil
}
func WriteFileFlagsFromTarHeader(path string, hdr *tar.Header) error {
return nil
}
func resetImmutable(path string, fi *os.FileInfo) error {
return nil
}

View File

@ -82,7 +82,7 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions
}
}
r := io.NopCloser(tarArchive)
r := tarArchive
if decompress {
decompressedArchive, err := archive.DecompressStream(tarArchive)
if err != nil {

View File

@ -6,11 +6,7 @@ import (
"github.com/containers/storage/pkg/archive"
)
func chroot(path string) error {
return nil
}
func invokeUnpack(decompressedArchive io.ReadCloser,
func invokeUnpack(decompressedArchive io.Reader,
dest string,
options *archive.TarOptions, root string) error {
return archive.Unpack(decompressedArchive, dest, options)

View File

@ -12,7 +12,7 @@ func chroot(path string) error {
return nil
}
func invokeUnpack(decompressedArchive io.ReadCloser,
func invokeUnpack(decompressedArchive io.Reader,
dest string,
options *archive.TarOptions, root string) error {
// Windows is different to Linux here because Windows does not support

View File

@ -27,13 +27,13 @@ func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions
tmpDir, err := os.MkdirTemp(os.Getenv("temp"), "temp-storage-extract")
if err != nil {
return 0, fmt.Errorf("ApplyLayer failed to create temp-storage-extract under %s. %s", dest, err)
return 0, fmt.Errorf("ApplyLayer failed to create temp-storage-extract under %s: %w", dest, err)
}
s, err := archive.UnpackLayer(dest, layer, options)
os.RemoveAll(tmpDir)
if err != nil {
return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %s", layer, dest, err)
return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %w", layer, dest, err)
}
return s, nil

View File

@ -1,3 +1,6 @@
//go:build !windows && !darwin
// +build !windows,!darwin
package chrootarchive
import jsoniter "github.com/json-iterator/go"

View File

@ -1,21 +1,10 @@
package chunked
import (
archivetar "archive/tar"
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"strconv"
"github.com/containerd/stargz-snapshotter/estargz"
"github.com/containers/storage/pkg/chunked/compressor"
"github.com/containers/storage/pkg/chunked/internal"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
digest "github.com/opencontainers/go-digest"
"github.com/vbatts/tar-split/archive/tar"
)
const (
@ -29,247 +18,6 @@ const (
TypeSymlink = internal.TypeSymlink
)
var typesToTar = map[string]byte{
TypeReg: tar.TypeReg,
TypeLink: tar.TypeLink,
TypeChar: tar.TypeChar,
TypeBlock: tar.TypeBlock,
TypeDir: tar.TypeDir,
TypeFifo: tar.TypeFifo,
TypeSymlink: tar.TypeSymlink,
}
func typeToTarType(t string) (byte, error) {
r, found := typesToTar[t]
if !found {
return 0, fmt.Errorf("unknown type: %v", t)
}
return r, nil
}
func isZstdChunkedFrameMagic(data []byte) bool {
if len(data) < 8 {
return false
}
return bytes.Equal(internal.ZstdChunkedFrameMagic, data[:8])
}
func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
// information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md
footerSize := int64(51)
if blobSize <= footerSize {
return nil, 0, errors.New("blob too small")
}
chunk := ImageSourceChunk{
Offset: uint64(blobSize - footerSize),
Length: uint64(footerSize),
}
parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
if err != nil {
return nil, 0, err
}
var reader io.ReadCloser
select {
case r := <-parts:
reader = r
case err := <-errs:
return nil, 0, err
}
defer reader.Close()
footer := make([]byte, footerSize)
if _, err := io.ReadFull(reader, footer); err != nil {
return nil, 0, err
}
/* Read the ToC offset:
- 10 bytes gzip header
- 2 bytes XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ"))
- 2 bytes Extra: SI1 = 'S', SI2 = 'G'
- 2 bytes Extra: LEN = 22 (16 hex digits + len("STARGZ"))
- 22 bytes Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC)
- 5 bytes flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0
- 8 bytes gzip footer
*/
tocOffset, err := strconv.ParseInt(string(footer[16:16+22-6]), 16, 64)
if err != nil {
return nil, 0, fmt.Errorf("parse ToC offset: %w", err)
}
size := int64(blobSize - footerSize - tocOffset)
// set a reasonable limit
if size > (1<<20)*50 {
return nil, 0, errors.New("manifest too big")
}
chunk = ImageSourceChunk{
Offset: uint64(tocOffset),
Length: uint64(size),
}
parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk})
if err != nil {
return nil, 0, err
}
var tocReader io.ReadCloser
select {
case r := <-parts:
tocReader = r
case err := <-errs:
return nil, 0, err
}
defer tocReader.Close()
r, err := pgzip.NewReader(tocReader)
if err != nil {
return nil, 0, err
}
defer r.Close()
aTar := archivetar.NewReader(r)
header, err := aTar.Next()
if err != nil {
return nil, 0, err
}
// set a reasonable limit
if header.Size > (1<<20)*50 {
return nil, 0, errors.New("manifest too big")
}
manifestUncompressed := make([]byte, header.Size)
if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil {
return nil, 0, err
}
manifestDigester := digest.Canonical.Digester()
manifestChecksum := manifestDigester.Hash()
if _, err := manifestChecksum.Write(manifestUncompressed); err != nil {
return nil, 0, err
}
d, err := digest.Parse(annotations[estargz.TOCJSONDigestAnnotation])
if err != nil {
return nil, 0, err
}
if manifestDigester.Digest() != d {
return nil, 0, errors.New("invalid manifest checksum")
}
return manifestUncompressed, tocOffset, nil
}
// readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream. The blob total size must
// be specified.
// This function uses the io.containers.zstd-chunked. annotations when specified.
func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
footerSize := int64(internal.FooterSizeSupported)
if blobSize <= footerSize {
return nil, 0, errors.New("blob too small")
}
manifestChecksumAnnotation := annotations[internal.ManifestChecksumKey]
if manifestChecksumAnnotation == "" {
return nil, 0, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey)
}
var offset, length, lengthUncompressed, manifestType uint64
if offsetMetadata := annotations[internal.ManifestInfoKey]; offsetMetadata != "" {
if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil {
return nil, 0, err
}
} else {
chunk := ImageSourceChunk{
Offset: uint64(blobSize - footerSize),
Length: uint64(footerSize),
}
parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
if err != nil {
return nil, 0, err
}
var reader io.ReadCloser
select {
case r := <-parts:
reader = r
case err := <-errs:
return nil, 0, err
}
footer := make([]byte, footerSize)
if _, err := io.ReadFull(reader, footer); err != nil {
return nil, 0, err
}
offset = binary.LittleEndian.Uint64(footer[0:8])
length = binary.LittleEndian.Uint64(footer[8:16])
lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
manifestType = binary.LittleEndian.Uint64(footer[24:32])
if !isZstdChunkedFrameMagic(footer[32:40]) {
return nil, 0, errors.New("invalid magic number")
}
}
if manifestType != internal.ManifestTypeCRFS {
return nil, 0, errors.New("invalid manifest type")
}
// set a reasonable limit
if length > (1<<20)*50 {
return nil, 0, errors.New("manifest too big")
}
if lengthUncompressed > (1<<20)*50 {
return nil, 0, errors.New("manifest too big")
}
chunk := ImageSourceChunk{
Offset: offset,
Length: length,
}
parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
if err != nil {
return nil, 0, err
}
var reader io.ReadCloser
select {
case r := <-parts:
reader = r
case err := <-errs:
return nil, 0, err
}
manifest := make([]byte, length)
if _, err := io.ReadFull(reader, manifest); err != nil {
return nil, 0, err
}
manifestDigester := digest.Canonical.Digester()
manifestChecksum := manifestDigester.Hash()
if _, err := manifestChecksum.Write(manifest); err != nil {
return nil, 0, err
}
d, err := digest.Parse(manifestChecksumAnnotation)
if err != nil {
return nil, 0, err
}
if manifestDigester.Digest() != d {
return nil, 0, errors.New("invalid manifest checksum")
}
decoder, err := zstd.NewReader(nil)
if err != nil {
return nil, 0, err
}
defer decoder.Close()
b := make([]byte, 0, lengthUncompressed)
if decoded, err := decoder.DecodeAll(manifest, b); err == nil {
return decoded, int64(offset), nil
}
return manifest, int64(offset), nil
}
// ZstdCompressor is a CompressorFunc for the zstd compression algorithm.
// Deprecated: Use pkg/chunked/compressor.ZstdCompressor.
func ZstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {

Some files were not shown because too many files have changed in this diff Show More