vendor: bump c/image to 373c52a9466f

[NO NEW TESTS NEEDED]

Signed-off-by: Aditya R <arajan@redhat.com>
This commit is contained in:
Aditya R 2023-11-01 23:28:29 +05:30
parent 0cd20090b2
commit 03419d6daa
No known key found for this signature in database
GPG Key ID: 8E5A8A19DF7C8673
123 changed files with 2387 additions and 1668 deletions

33
go.mod
View File

@ -16,11 +16,11 @@ require (
github.com/containers/common v0.56.1-0.20231026130642-78e0a90c7c2f
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/gvisor-tap-vsock v0.7.1
github.com/containers/image/v5 v5.28.0
github.com/containers/image/v5 v5.28.1-0.20231101173728-373c52a9466f
github.com/containers/libhvee v0.4.1-0.20231012183749-e51be96b4854
github.com/containers/ocicrypt v1.1.8
github.com/containers/ocicrypt v1.1.9
github.com/containers/psgo v1.8.0
github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff
github.com/containers/storage v1.50.3-0.20231101112703-6e72f11598fb
github.com/coreos/go-systemd/v22 v22.5.0
github.com/coreos/stream-metadata-go v0.4.3
github.com/crc-org/vfkit v0.1.2-0.20231030102423-f3c783d34420
@ -51,7 +51,7 @@ require (
github.com/onsi/gomega v1.29.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc5
github.com/opencontainers/runc v1.1.9
github.com/opencontainers/runc v1.1.10
github.com/opencontainers/runtime-spec v1.1.1-0.20230922153023-c0e90434df2a
github.com/opencontainers/runtime-tools v0.9.1-0.20230914150019-408c51e934dc
github.com/opencontainers/selinux v1.11.0
@ -67,7 +67,7 @@ require (
github.com/vbauerster/mpb/v8 v8.6.2
github.com/vishvananda/netlink v1.2.1-beta.2
go.etcd.io/bbolt v1.3.8
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/net v0.17.0
golang.org/x/sync v0.4.0
golang.org/x/sys v0.13.0
@ -95,10 +95,10 @@ require (
github.com/containerd/cgroups/v3 v3.0.2 // indirect
github.com/containerd/containerd v1.7.7 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
github.com/containers/luksy v0.0.0-20230912175440-6df88cb7f0dd // indirect
github.com/coreos/go-oidc/v3 v3.6.0 // indirect
github.com/coreos/go-oidc/v3 v3.7.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
@ -144,7 +144,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.1 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/fs v0.1.0 // indirect
@ -179,14 +179,13 @@ require (
github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
github.com/segmentio/ksuid v1.0.4 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sigstore/fulcio v1.4.0 // indirect
github.com/sigstore/fulcio v1.4.3 // indirect
github.com/sigstore/rekor v1.2.2 // indirect
github.com/sigstore/sigstore v1.7.3 // indirect
github.com/sigstore/sigstore v1.7.5 // indirect
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
github.com/sylabs/sif/v2 v2.13.0 // indirect
github.com/sylabs/sif/v2 v2.15.0 // indirect
github.com/tchap/go-patricia/v2 v2.3.1 // indirect
github.com/theupdateframework/go-tuf v0.5.2 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
@ -205,11 +204,11 @@ require (
golang.org/x/arch v0.5.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/tools v0.13.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/grpc v1.57.1 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
google.golang.org/grpc v1.58.3 // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect

72
go.sum
View File

@ -228,8 +228,8 @@ github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oM
github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M=
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
@ -262,8 +262,8 @@ github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6J
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/gvisor-tap-vsock v0.7.1 h1:+Rc+sOPplrkQb/BUXeN0ug8TxjgyrIqo/9P/eNS2A4c=
github.com/containers/gvisor-tap-vsock v0.7.1/go.mod h1:WSSsjcuYZkvP8i0J+Ht3LF8yvysn3krD5zxQ74wz7y0=
github.com/containers/image/v5 v5.28.0 h1:H4cWbdI88UA/mDb6SxMo3IxpmS1BSs/Kifvhwt9g048=
github.com/containers/image/v5 v5.28.0/go.mod h1:9aPnNkwHNHgGl9VlQxXEshvmOJRbdRAc1rNDD6sP2eU=
github.com/containers/image/v5 v5.28.1-0.20231101173728-373c52a9466f h1:x79xiC/Zs7yRzCWCT/fuf8J8LALTzVHzGT9T0HEx9FQ=
github.com/containers/image/v5 v5.28.1-0.20231101173728-373c52a9466f/go.mod h1:7+h9aIQgB6YzWxFzKAAYQ0CQZS0ks/bc+FMZQTJFoN8=
github.com/containers/libhvee v0.4.1-0.20231012183749-e51be96b4854 h1:9pHtBDAO1ZE0Cwhn3rfp7CfqpfeaYllG2o6wuDdxsa8=
github.com/containers/libhvee v0.4.1-0.20231012183749-e51be96b4854/go.mod h1:3lTcwI2g7qe8Ekgk9hdDxQeT9KrqXPilQvxJfIJp8TQ=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
@ -273,20 +273,20 @@ github.com/containers/luksy v0.0.0-20230912175440-6df88cb7f0dd/go.mod h1:p3x2uBi
github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
github.com/containers/ocicrypt v1.1.8 h1:saSBF0/8DyPUjzcxMVzL2OBUWCkvRvqIm75pu0ADSZk=
github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/LFMtH0FIRt34=
github.com/containers/ocicrypt v1.1.9 h1:2Csfba4jse85Raxk5HIyEk8OwZNjRvfkhEGijOjIdEM=
github.com/containers/ocicrypt v1.1.9/go.mod h1:dTKx1918d8TDkxXvarscpNVY+lyPakPNFN4jwA9GBys=
github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY=
github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc=
github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s=
github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff h1:/z85oN77tAw5UUEzymFH73nHIgNtGeL9YednTut9E8M=
github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff/go.mod h1:UMgAdUgejmMKJQqeRE+0X/3GP9O6rXRm+E4uS7QrFgk=
github.com/containers/storage v1.50.3-0.20231101112703-6e72f11598fb h1:g1IJUHmHZuHa1YPvIiYjWrhysb+qEiiImA8p8mENhiE=
github.com/containers/storage v1.50.3-0.20231101112703-6e72f11598fb/go.mod h1:LpKczONfqahkVHFdZGPUg/xYZVjd/qqisRu0TkO4u8k=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-oidc/v3 v3.6.0 h1:AKVxfYw1Gmkn/w96z0DbT/B/xFnzTd3MkZvWLjF4n/o=
github.com/coreos/go-oidc/v3 v3.6.0/go.mod h1:ZpHUsHBucTUj6WOkrP4E20UPynbLZzhTQ1XKCXkxyPc=
github.com/coreos/go-oidc/v3 v3.7.0 h1:FTdj0uexT4diYIPlF4yoFVI5MRO1r5+SEcIpEw9vC0o=
github.com/coreos/go-oidc/v3 v3.7.0/go.mod h1:yQzSCqBnK3e6Fs5l+f5i0F8Kwf0zpH9bPEsbY00KanM=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@ -476,7 +476,7 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-rod/rod v0.114.3 h1:gAUT2Bc2wy0tQL5KEet05HNDvmndaHAGCjQ01TB2efA=
github.com/go-rod/rod v0.114.4 h1:FpkNFukjCuZLwnoLs+S9aCL95o/EMec6M+41UmvQay8=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
@ -696,8 +696,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@ -876,6 +876,7 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f h1:/UDgs8FGMqwnHagNDPGOlts35QkhAZ8by3DR7nMih7M=
github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc=
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
@ -905,13 +906,13 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@ -930,7 +931,7 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
@ -964,12 +965,12 @@ github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sigstore/fulcio v1.4.0 h1:05+k8BFvwTQzfCkVxESWzCN4b70KIRliGYz0Upmdrs8=
github.com/sigstore/fulcio v1.4.0/go.mod h1:wcjlktbhoy6+ZTxO3yXpvqUxsLV+JEH4FF3a5Jz4VPI=
github.com/sigstore/fulcio v1.4.3 h1:9JcUCZjjVhRF9fmhVuz6i1RyhCc/EGCD7MOl+iqCJLQ=
github.com/sigstore/fulcio v1.4.3/go.mod h1:BQPWo7cfxmJwgaHlphUHUpFkp5+YxeJes82oo39m5og=
github.com/sigstore/rekor v1.2.2 h1:5JK/zKZvcQpL/jBmHvmFj3YbpDMBQnJQ6ygp8xdF3bY=
github.com/sigstore/rekor v1.2.2/go.mod h1:FGnWBGWzeNceJnp0x9eDFd41mI8aQqCjj+Zp0IEs0Qg=
github.com/sigstore/sigstore v1.7.3 h1:HVVTfrMezJeLyl2xhJ8edzkrEGBa4KxjQZB4FlQ4JLU=
github.com/sigstore/sigstore v1.7.3/go.mod h1:cl0c7Dtg3MM3c13L8pqqrfrmBa0eM3POcdtBepjylmw=
github.com/sigstore/sigstore v1.7.5 h1:ij55dBhLwjICmLTBJZm7SqoQLdsu/oowDanACcJNs48=
github.com/sigstore/sigstore v1.7.5/go.mod h1:9OCmYWhzuq/G4e1cy9m297tuMRJ1LExyrXY3ZC3Zt/s=
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@ -1026,8 +1027,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/sylabs/sif/v2 v2.13.0 h1:dK/PQ/ohLAA4hptbjNuU0qoqkJ9Kl07hiSHArMNSKsQ=
github.com/sylabs/sif/v2 v2.13.0/go.mod h1:qEFrmE29XNbW2uyBagTsw9dgM82MwsckNYUFPweF2ek=
github.com/sylabs/sif/v2 v2.15.0 h1:Nv0tzksFnoQiQ2eUwpAis9nVqEu4c3RcNSxX8P3Cecw=
github.com/sylabs/sif/v2 v2.15.0/go.mod h1:X1H7eaPz6BAxA84POMESXoXfTqgAnLQkujyF/CQFWTc=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
@ -1036,8 +1037,6 @@ github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ
github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes=
github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=
github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA=
github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0=
@ -1183,8 +1182,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1266,8 +1265,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -1394,6 +1393,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@ -1459,8 +1459,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1483,8 +1483,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -1511,8 +1511,8 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@ -1531,8 +1531,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg=
google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

View File

@ -436,9 +436,8 @@ func importTar(in io.ReaderAt) (*tarFile, error) {
if err != nil {
if err == io.EOF {
break
} else {
return nil, fmt.Errorf("failed to parse tar file, %w", err)
}
return nil, fmt.Errorf("failed to parse tar file, %w", err)
}
switch cleanEntryName(h.Name) {
case PrefetchLandmark, NoPrefetchLandmark:

View File

@ -70,7 +70,7 @@ func (d *bpDecryptionStepData) updateCryptoOperation(operation *types.LayerCrypt
}
}
// bpdData contains data that the copy pipeline needs about the encryption step.
// bpEncryptionStepData contains data that the copy pipeline needs about the encryption step.
type bpEncryptionStepData struct {
encrypting bool // We are actually encrypting the stream
finalizer ocicrypt.EncryptLayerFinalizer

View File

@ -340,7 +340,7 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
if err != nil {
return nil, err
}
sigs = append(sigs, newSigs...)
sigs = append(slices.Clone(sigs), newSigs...)
c.Printf("Storing list signatures\n")
if err := c.dest.PutSignaturesWithFormat(ctx, sigs, nil); err != nil {

View File

@ -277,7 +277,7 @@ func (c *copier) copySingleImage(ctx context.Context, unparsedImage *image.Unpar
if err != nil {
return copySingleImageResult{}, err
}
sigs = append(sigs, newSigs...)
sigs = append(slices.Clone(sigs), newSigs...)
if len(sigs) > 0 {
c.Printf("Storing signatures\n")
@ -380,8 +380,9 @@ func (ic *imageCopier) compareImageDestinationManifestEqual(ctx context.Context,
compressionAlgos := set.New[string]()
for _, srcInfo := range ic.src.LayerInfos() {
compression := compressionAlgorithmFromMIMEType(srcInfo)
compressionAlgos.Add(compression.Name())
if c := compressionAlgorithmFromMIMEType(srcInfo); c != nil {
compressionAlgos.Add(c.Name())
}
}
algos, err := algorithmsByNames(compressionAlgos.Values())
@ -743,7 +744,9 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
uploadedBlob, err := ic.c.dest.PutBlobPartial(ctx, &proxy, srcInfo, ic.c.blobInfoCache)
if err == nil {
if srcInfo.Size != -1 {
bar.SetRefill(srcInfo.Size - bar.Current())
refill := srcInfo.Size - bar.Current()
bar.SetCurrent(srcInfo.Size)
bar.SetRefill(refill)
}
bar.mark100PercentComplete()
hideProgressBar = false

View File

@ -2,6 +2,7 @@ package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
@ -85,12 +86,40 @@ func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeRe
}
}()
err = imageLoad(ctx, c, reader)
}
// imageLoad accepts tar stream on reader and sends it to c
func imageLoad(ctx context.Context, c *client.Client, reader *io.PipeReader) error {
resp, err := c.ImageLoad(ctx, reader, true)
if err != nil {
err = fmt.Errorf("saving image to docker engine: %w", err)
return
return fmt.Errorf("starting a load operation in docker engine: %w", err)
}
defer resp.Body.Close()
// jsonError and jsonMessage are small subsets of docker/docker/pkg/jsonmessage.JSONError and JSONMessage,
// copied here to minimize dependencies.
type jsonError struct {
Message string `json:"message,omitempty"`
}
type jsonMessage struct {
Error *jsonError `json:"errorDetail,omitempty"`
}
dec := json.NewDecoder(resp.Body)
for {
var msg jsonMessage
if err := dec.Decode(&msg); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("parsing docker load progress: %w", err)
}
if msg.Error != nil {
return fmt.Errorf("docker engine reported: %s", msg.Error.Message)
}
}
return nil // No error reported = success
}
// DesiredLayerCompression indicates if layers must be compressed, decompressed or preserved

View File

@ -24,6 +24,7 @@ import (
"github.com/docker/distribution/registry/api/errcode"
dockerChallenge "github.com/docker/distribution/registry/client/auth/challenge"
"golang.org/x/exp/slices"
)
// errNoErrorsInBody is returned when an HTTP response body parses to an empty
@ -105,7 +106,7 @@ func makeErrorList(err error) []error {
}
func mergeErrors(err1, err2 error) error {
return errcode.Errors(append(makeErrorList(err1), makeErrorList(err2)...))
return errcode.Errors(append(slices.Clone(makeErrorList(err1)), makeErrorList(err2)...))
}
// handleErrorResponse returns error parsed from HTTP response for an

View File

@ -363,6 +363,11 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima
hostname := registry
if registry == dockerHostname {
hostname = dockerV1Hostname
// A search term of library/foo does not find the library/foo image on the docker.io servers,
// which is surprising - and that Docker is modifying the search term client-side this same way,
// and it seems convenient to do the same thing.
// Read more here: https://github.com/containers/image/pull/2133#issue-1928524334
image = strings.TrimPrefix(image, "library/")
}
client, err := newDockerClient(sys, hostname, registry)

View File

@ -137,7 +137,7 @@ func (d *dockerImageDestination) PutBlobWithOptions(ctx context.Context, stream
// If requested, precompute the blob digest to prevent uploading layers that already exist on the registry.
// This functionality is particularly useful when BlobInfoCache has not been populated with compressed digests,
// the source blob is uncompressed, and the destination blob is being compressed "on the fly".
if inputInfo.Digest == "" && d.c.sys.DockerRegistryPushPrecomputeDigests {
if inputInfo.Digest == "" && d.c.sys != nil && d.c.sys.DockerRegistryPushPrecomputeDigests {
logrus.Debugf("Precomputing digest layer for %s", reference.Path(d.ref.ref))
streamCopy, cleanup, err := streamdigest.ComputeBlobInfo(d.c.sys, stream, &inputInfo)
if err != nil {
@ -341,39 +341,58 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context,
// Then try reusing blobs from other locations.
candidates := options.Cache.CandidateLocations2(d.ref.Transport(), bicTransportScope(d.ref), info.Digest, options.CanSubstitute)
for _, candidate := range candidates {
candidateRepo, err := parseBICLocationReference(candidate.Location)
if err != nil {
logrus.Debugf("Error parsing BlobInfoCache location reference: %s", err)
continue
}
var err error
compressionOperation, compressionAlgorithm, err := blobinfocache.OperationAndAlgorithmForCompressor(candidate.CompressorName)
if err != nil {
logrus.Debugf("OperationAndAlgorithmForCompressor Failed: %v", err)
continue
}
var candidateRepo reference.Named
if !candidate.UnknownLocation {
candidateRepo, err = parseBICLocationReference(candidate.Location)
if err != nil {
logrus.Debugf("Error parsing BlobInfoCache location reference: %s", err)
continue
}
}
if !impl.BlobMatchesRequiredCompression(options, compressionAlgorithm) {
requiredCompression := "nil"
if compressionAlgorithm != nil {
requiredCompression = compressionAlgorithm.Name()
}
logrus.Debugf("Ignoring candidate blob %s as reuse candidate due to compression mismatch ( %s vs %s ) in %s", candidate.Digest.String(), options.RequiredCompression.Name(), requiredCompression, candidateRepo.Name())
if !candidate.UnknownLocation {
logrus.Debugf("Ignoring candidate blob %s as reuse candidate due to compression mismatch ( %s vs %s ) in %s", candidate.Digest.String(), options.RequiredCompression.Name(), requiredCompression, candidateRepo.Name())
} else {
logrus.Debugf("Ignoring candidate blob %s as reuse candidate due to compression mismatch ( %s vs %s ) with no location match, checking current repo", candidate.Digest.String(), options.RequiredCompression.Name(), requiredCompression)
}
continue
}
if candidate.CompressorName != blobinfocache.Uncompressed {
logrus.Debugf("Trying to reuse cached location %s compressed with %s in %s", candidate.Digest.String(), candidate.CompressorName, candidateRepo.Name())
if !candidate.UnknownLocation {
if candidate.CompressorName != blobinfocache.Uncompressed {
logrus.Debugf("Trying to reuse blob with cached digest %s compressed with %s in destination repo %s", candidate.Digest.String(), candidate.CompressorName, candidateRepo.Name())
} else {
logrus.Debugf("Trying to reuse blob with cached digest %s in destination repo %s", candidate.Digest.String(), candidateRepo.Name())
}
// Sanity checks:
if reference.Domain(candidateRepo) != reference.Domain(d.ref.ref) {
// OCI distribution spec 1.1 allows mounting blobs without specifying the source repo
// (the "from" parameter); in that case we might try to use these candidates as well.
//
// OTOH that would mean we cant do the “blobExists” check, and if there is no match
// we could get an upload request that we would have to cancel.
logrus.Debugf("... Internal error: domain %s does not match destination %s", reference.Domain(candidateRepo), reference.Domain(d.ref.ref))
continue
}
} else {
logrus.Debugf("Trying to reuse cached location %s with no compression in %s", candidate.Digest.String(), candidateRepo.Name())
}
// Sanity checks:
if reference.Domain(candidateRepo) != reference.Domain(d.ref.ref) {
// OCI distribution spec 1.1 allows mounting blobs without specifying the source repo
// (the "from" parameter); in that case we might try to use these candidates as well.
//
// OTOH that would mean we cant do the “blobExists” check, and if there is no match
// we could get an upload request that we would have to cancel.
logrus.Debugf("... Internal error: domain %s does not match destination %s", reference.Domain(candidateRepo), reference.Domain(d.ref.ref))
continue
if candidate.CompressorName != blobinfocache.Uncompressed {
logrus.Debugf("Trying to reuse blob with cached digest %s compressed with %s with no location match, checking current repo", candidate.Digest.String(), candidate.CompressorName)
} else {
logrus.Debugf("Trying to reuse blob with cached digest %s in destination repo with no location match, checking current repo", candidate.Digest.String())
}
// This digest is a known variant of this blob but we dont
// have a recorded location in this registry, lets try looking
// for it in the current repo.
candidateRepo = reference.TrimNamed(d.ref.ref)
}
if candidateRepo.Name() == d.ref.ref.Name() && candidate.Digest == info.Digest {
logrus.Debug("... Already tried the primary destination")
@ -688,6 +707,10 @@ func (d *dockerImageDestination) putSignaturesToSigstoreAttachments(ctx context.
}
}
// To make sure we can safely append to the slices of ociManifest, without adding a remote dependency on the code that creates it.
ociManifest.Layers = slices.Clone(ociManifest.Layers)
// We dont need to ^^^ for ociConfig.RootFS.DiffIDs because we have created it empty ourselves, and json.Unmarshal is documented to append() to
// the slice in the original object (or in a newly allocated object).
for _, sig := range signatures {
mimeType := sig.UntrustedMIMEType()
payloadBlob := sig.UntrustedPayload()

View File

@ -2,6 +2,8 @@ package image
import (
"github.com/containers/image/v5/internal/image"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/internal/unparsedimage"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
)
@ -17,3 +19,23 @@ type UnparsedImage = image.UnparsedImage
func UnparsedInstance(src types.ImageSource, instanceDigest *digest.Digest) *UnparsedImage {
return image.UnparsedInstance(src, instanceDigest)
}
// unparsedWithRef wraps a private.UnparsedImage, claiming another replacementRef
type unparsedWithRef struct {
private.UnparsedImage
ref types.ImageReference
}
func (uwr *unparsedWithRef) Reference() types.ImageReference {
return uwr.ref
}
// UnparsedInstanceWithReference returns a types.UnparsedImage for wrappedInstance which claims to be a replacementRef.
// This is useful for combining image data with other reference values, e.g. to check signatures on a locally-pulled image
// based on a remote-registry policy.
func UnparsedInstanceWithReference(wrappedInstance types.UnparsedImage, replacementRef types.ImageReference) types.UnparsedImage {
return &unparsedWithRef{
UnparsedImage: unparsedimage.FromPublic(wrappedInstance),
ref: replacementRef,
}
}

View File

@ -32,7 +32,7 @@ type BlobInfoCache2 interface {
// otherwise the cache could be poisoned and cause us to make incorrect edits to type
// information in a manifest.
RecordDigestCompressorName(anyDigest digest.Digest, compressorName string)
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations (if known)
// that could possibly be reused within the specified (transport scope) (if they still
// exist, which is not guaranteed).
//
@ -46,7 +46,8 @@ type BlobInfoCache2 interface {
// BICReplacementCandidate2 is an item returned by BlobInfoCache2.CandidateLocations2.
type BICReplacementCandidate2 struct {
Digest digest.Digest
CompressorName string // either the Name() of a known pkg/compression.Algorithm, or Uncompressed or UnknownCompression
Location types.BICLocationReference
Digest digest.Digest
CompressorName string // either the Name() of a known pkg/compression.Algorithm, or Uncompressed or UnknownCompression
UnknownLocation bool // is true when `Location` for this blob is not set
Location types.BICLocationReference // not set if UnknownLocation is set to `true`
}

View File

@ -196,14 +196,12 @@ func (m *manifestOCI1) convertToManifestSchema2Generic(ctx context.Context, opti
return m.convertToManifestSchema2(ctx, options)
}
// prepareLayerDecryptEditsIfNecessary checks if options requires layer decryptions.
// layerEditsOfOCIOnlyFeatures checks if options requires some layer edits to be done before converting to a Docker format.
// If not, it returns (nil, nil).
// If decryption is required, it returns a set of edits to provide to OCI1.UpdateLayerInfos,
// and edits *options to not try decryption again.
func (m *manifestOCI1) prepareLayerDecryptEditsIfNecessary(options *types.ManifestUpdateOptions) ([]types.BlobInfo, error) {
if options == nil || !slices.ContainsFunc(options.LayerInfos, func(info types.BlobInfo) bool {
return info.CryptoOperation == types.Decrypt
}) {
func (m *manifestOCI1) layerEditsOfOCIOnlyFeatures(options *types.ManifestUpdateOptions) ([]types.BlobInfo, error) {
if options == nil || options.LayerInfos == nil {
return nil, nil
}
@ -212,19 +210,35 @@ func (m *manifestOCI1) prepareLayerDecryptEditsIfNecessary(options *types.Manife
return nil, fmt.Errorf("preparing to decrypt before conversion: %d layers vs. %d layer edits", len(originalInfos), len(options.LayerInfos))
}
res := slices.Clone(originalInfos) // Start with a full copy so that we don't forget to copy anything: use the current data in full unless we intentionaly deviate.
updatedEdits := slices.Clone(options.LayerInfos)
for i, info := range options.LayerInfos {
if info.CryptoOperation == types.Decrypt {
res[i].CryptoOperation = types.Decrypt
updatedEdits[i].CryptoOperation = types.PreserveOriginalCrypto // Don't try to decrypt in a schema[12] manifest later, that would fail.
ociOnlyEdits := slices.Clone(originalInfos) // Start with a full copy so that we don't forget to copy anything: use the current data in full unless we intentionally deviate.
laterEdits := slices.Clone(options.LayerInfos)
needsOCIOnlyEdits := false
for i, edit := range options.LayerInfos {
// Unless determined otherwise, don't do any compression-related MIME type conversions. m.LayerInfos() should not set these edit instructions, but be explicit.
ociOnlyEdits[i].CompressionOperation = types.PreserveOriginal
ociOnlyEdits[i].CompressionAlgorithm = nil
if edit.CryptoOperation == types.Decrypt {
needsOCIOnlyEdits = true // Encrypted types must be removed before conversion because they cant be represented in Docker schemas
ociOnlyEdits[i].CryptoOperation = types.Decrypt
laterEdits[i].CryptoOperation = types.PreserveOriginalCrypto // Don't try to decrypt in a schema[12] manifest later, that would fail.
}
if originalInfos[i].MediaType == imgspecv1.MediaTypeImageLayerZstd ||
originalInfos[i].MediaType == imgspecv1.MediaTypeImageLayerNonDistributableZstd { //nolint:staticcheck // NonDistributable layers are deprecated, but we want to continue to support manipulating pre-existing images.
needsOCIOnlyEdits = true // Zstd MIME types must be removed before conversion because they cant be represented in Docker schemas.
ociOnlyEdits[i].CompressionOperation = edit.CompressionOperation
ociOnlyEdits[i].CompressionAlgorithm = edit.CompressionAlgorithm
laterEdits[i].CompressionOperation = types.PreserveOriginal
laterEdits[i].CompressionAlgorithm = nil
}
// Don't do any compression-related MIME type conversions. m.LayerInfos() should not set these edit instructions, but be explicit.
res[i].CompressionOperation = types.PreserveOriginal
res[i].CompressionAlgorithm = nil
}
options.LayerInfos = updatedEdits
return res, nil
if !needsOCIOnlyEdits {
return nil, nil
}
options.LayerInfos = laterEdits
return ociOnlyEdits, nil
}
// convertToManifestSchema2 returns a genericManifest implementation converted to manifest.DockerV2Schema2MediaType.
@ -238,15 +252,15 @@ func (m *manifestOCI1) convertToManifestSchema2(_ context.Context, options *type
// Mostly we first make a format conversion, and _afterwards_ do layer edits. But first we need to do the layer edits
// which remove OCI-specific features, because trying to convert those layers would fail.
// So, do the layer updates for decryption.
// So, do the layer updates for decryption, and for conversions from Zstd.
ociManifest := m.m
layerDecryptEdits, err := m.prepareLayerDecryptEditsIfNecessary(options)
ociOnlyEdits, err := m.layerEditsOfOCIOnlyFeatures(options)
if err != nil {
return nil, err
}
if layerDecryptEdits != nil {
if ociOnlyEdits != nil {
ociManifest = manifest.OCI1Clone(ociManifest)
if err := ociManifest.UpdateLayerInfos(layerDecryptEdits); err != nil {
if err := ociManifest.UpdateLayerInfos(ociOnlyEdits); err != nil {
return nil, err
}
}
@ -275,9 +289,8 @@ func (m *manifestOCI1) convertToManifestSchema2(_ context.Context, options *type
layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
case imgspecv1.MediaTypeImageLayerZstd:
return nil, fmt.Errorf("Error during manifest conversion: %q: zstd compression is not supported for docker images", layers[idx].MediaType)
// FIXME: s/Zsdt/Zstd/ after ocicrypt with https://github.com/containers/ocicrypt/pull/91 is released
case ociencspec.MediaTypeLayerEnc, ociencspec.MediaTypeLayerGzipEnc, ociencspec.MediaTypeLayerZstdEnc,
ociencspec.MediaTypeLayerNonDistributableEnc, ociencspec.MediaTypeLayerNonDistributableGzipEnc, ociencspec.MediaTypeLayerNonDistributableZsdtEnc:
ociencspec.MediaTypeLayerNonDistributableEnc, ociencspec.MediaTypeLayerNonDistributableGzipEnc, ociencspec.MediaTypeLayerNonDistributableZstdEnc:
return nil, fmt.Errorf("during manifest conversion: encrypted layers (%q) are not supported in docker images", layers[idx].MediaType)
default:
return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", layers[idx].MediaType)

View File

@ -133,7 +133,9 @@ func (index *Schema2ListPublic) editInstances(editInstances []ListEdit) error {
}
}
if len(addedEntries) != 0 {
index.Manifests = append(index.Manifests, addedEntries...)
// slices.Clone() here to ensure a private backing array;
// an external caller could have manually created Schema2ListPublic with a slice with extra capacity.
index.Manifests = append(slices.Clone(index.Manifests), addedEntries...)
}
return nil
}

View File

@ -167,7 +167,9 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error {
}
}
if len(addedEntries) != 0 {
index.Manifests = append(index.Manifests, addedEntries...)
// slices.Clone() here to ensure the slice uses a private backing array;
// an external caller could have manually created OCI1IndexPublic with a slice with extra capacity.
index.Manifests = append(slices.Clone(index.Manifests), addedEntries...)
}
if len(addedEntries) != 0 || updatedAnnotations {
slices.SortStableFunc(index.Manifests, func(a, b imgspecv1.Descriptor) int {
@ -220,7 +222,7 @@ func (ic instanceCandidate) isPreferredOver(other *instanceCandidate, preferGzip
case ic.manifestPosition != other.manifestPosition:
return ic.manifestPosition < other.manifestPosition
}
panic("internal error: invalid comparision between two candidates") // This should not be reachable because in all calls we make, the two candidates differ at least in manifestPosition.
panic("internal error: invalid comparison between two candidates") // This should not be reachable because in all calls we make, the two candidates differ at least in manifestPosition.
}
// chooseInstance is a private equivalent to ChooseInstanceByCompression,

View File

@ -28,6 +28,18 @@ func (e ImageNotFoundError) Error() string {
return fmt.Sprintf("no descriptor found for reference %q", e.ref.image)
}
// ArchiveFileNotFoundError occurs when the archive file does not exist.
type ArchiveFileNotFoundError struct {
// ref is the image reference
ref ociArchiveReference
// path is the file path that was not present
path string
}
func (e ArchiveFileNotFoundError) Error() string {
return fmt.Sprintf("archive file not found: %q", e.path)
}
type ociArchiveImageSource struct {
impl.Compat

View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"strings"
@ -171,18 +172,24 @@ func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error)
// creates the temporary directory and copies the tarred content to it
func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) {
src := ref.resolvedFile
arch, err := os.Open(src)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return tempDirOCIRef{}, ArchiveFileNotFoundError{ref: ref, path: src}
} else {
return tempDirOCIRef{}, err
}
}
defer arch.Close()
tempDirRef, err := createOCIRef(sys, ref.image)
if err != nil {
return tempDirOCIRef{}, fmt.Errorf("creating oci reference: %w", err)
}
src := ref.resolvedFile
dst := tempDirRef.tempDirectory
// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
arch, err := os.Open(src)
if err != nil {
return tempDirOCIRef{}, err
}
defer arch.Close()
if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return tempDirOCIRef{}, fmt.Errorf("deleting temp directory %q: %w", tempDirRef.tempDirectory, err)

View File

@ -0,0 +1,240 @@
package layout
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"github.com/containers/image/v5/internal/set"
"github.com/containers/image/v5/types"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
)
// DeleteImage deletes the named image from the directory, if supported.
func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
sharedBlobsDir := ""
if sys != nil && sys.OCISharedBlobDirPath != "" {
sharedBlobsDir = sys.OCISharedBlobDirPath
}
descriptor, descriptorIndex, err := ref.getManifestDescriptor()
if err != nil {
return err
}
var blobsUsedByImage map[digest.Digest]int
switch descriptor.MediaType {
case imgspecv1.MediaTypeImageManifest:
blobsUsedByImage, err = ref.getBlobsUsedInSingleImage(&descriptor, sharedBlobsDir)
case imgspecv1.MediaTypeImageIndex:
blobsUsedByImage, err = ref.getBlobsUsedInImageIndex(&descriptor, sharedBlobsDir)
default:
return fmt.Errorf("unsupported mediaType in index: %q", descriptor.MediaType)
}
if err != nil {
return err
}
blobsToDelete, err := ref.getBlobsToDelete(blobsUsedByImage, sharedBlobsDir)
if err != nil {
return err
}
err = ref.deleteBlobs(blobsToDelete)
if err != nil {
return err
}
return ref.deleteReferenceFromIndex(descriptorIndex)
}
func (ref ociReference) getBlobsUsedInSingleImage(descriptor *imgspecv1.Descriptor, sharedBlobsDir string) (map[digest.Digest]int, error) {
manifest, err := ref.getManifest(descriptor, sharedBlobsDir)
if err != nil {
return nil, err
}
blobsUsedInManifest := ref.getBlobsUsedInManifest(manifest)
blobsUsedInManifest[descriptor.Digest]++ // Add the current manifest to the list of blobs used by this reference
return blobsUsedInManifest, nil
}
func (ref ociReference) getBlobsUsedInImageIndex(descriptor *imgspecv1.Descriptor, sharedBlobsDir string) (map[digest.Digest]int, error) {
blobPath, err := ref.blobPath(descriptor.Digest, sharedBlobsDir)
if err != nil {
return nil, err
}
index, err := parseIndex(blobPath)
if err != nil {
return nil, err
}
blobsUsedInImageRefIndex := make(map[digest.Digest]int)
err = ref.addBlobsUsedInIndex(blobsUsedInImageRefIndex, index, sharedBlobsDir)
if err != nil {
return nil, err
}
blobsUsedInImageRefIndex[descriptor.Digest]++ // Add the nested index in the list of blobs used by this reference
return blobsUsedInImageRefIndex, nil
}
// Updates a map of digest with the usage count, so a blob that is referenced three times will have 3 in the map
func (ref ociReference) addBlobsUsedInIndex(destination map[digest.Digest]int, index *imgspecv1.Index, sharedBlobsDir string) error {
for _, descriptor := range index.Manifests {
destination[descriptor.Digest]++
switch descriptor.MediaType {
case imgspecv1.MediaTypeImageManifest:
manifest, err := ref.getManifest(&descriptor, sharedBlobsDir)
if err != nil {
return err
}
for digest, count := range ref.getBlobsUsedInManifest(manifest) {
destination[digest] += count
}
case imgspecv1.MediaTypeImageIndex:
blobPath, err := ref.blobPath(descriptor.Digest, sharedBlobsDir)
if err != nil {
return err
}
index, err := parseIndex(blobPath)
if err != nil {
return err
}
err = ref.addBlobsUsedInIndex(destination, index, sharedBlobsDir)
if err != nil {
return err
}
default:
return fmt.Errorf("unsupported mediaType in index: %q", descriptor.MediaType)
}
}
return nil
}
func (ref ociReference) getBlobsUsedInManifest(manifest *imgspecv1.Manifest) map[digest.Digest]int {
blobsUsedInManifest := make(map[digest.Digest]int, 0)
blobsUsedInManifest[manifest.Config.Digest]++
for _, layer := range manifest.Layers {
blobsUsedInManifest[layer.Digest]++
}
return blobsUsedInManifest
}
// This takes in a map of the digest and their usage count in the manifest to be deleted
// It will compare it to the digest usage in the root index, and return a set of the blobs that can be safely deleted
func (ref ociReference) getBlobsToDelete(blobsUsedByDescriptorToDelete map[digest.Digest]int, sharedBlobsDir string) (*set.Set[digest.Digest], error) {
rootIndex, err := ref.getIndex()
if err != nil {
return nil, err
}
blobsUsedInRootIndex := make(map[digest.Digest]int)
err = ref.addBlobsUsedInIndex(blobsUsedInRootIndex, rootIndex, sharedBlobsDir)
if err != nil {
return nil, err
}
blobsToDelete := set.New[digest.Digest]()
for digest, count := range blobsUsedInRootIndex {
if count-blobsUsedByDescriptorToDelete[digest] == 0 {
blobsToDelete.Add(digest)
}
}
return blobsToDelete, nil
}
// This transport never generates layouts where blobs for an image are both in the local blobs directory
// and the shared one; its either one or the other, depending on how OCISharedBlobDirPath is set.
//
// But we cant correctly compute use counts for OCISharedBlobDirPath (because we don't know what
// the other layouts sharing that directory are, and we might not even have permission to read them),
// so we cant really delete any blobs in that case.
// Checking the _local_ blobs directory, and deleting blobs from there, doesn't really hurt,
// in case the layout was created using some other tool or without OCISharedBlobDirPath set, so let's silently
// check for local blobs (but we should make no noise if the blobs are actually in the shared directory).
//
// So, NOTE: the blobPath() call below hard-codes "" even in calls where OCISharedBlobDirPath is set
func (ref ociReference) deleteBlobs(blobsToDelete *set.Set[digest.Digest]) error {
for _, digest := range blobsToDelete.Values() {
blobPath, err := ref.blobPath(digest, "") //Only delete in the local directory, see comment above
if err != nil {
return err
}
err = deleteBlob(blobPath)
if err != nil {
return err
}
}
return nil
}
func deleteBlob(blobPath string) error {
logrus.Debug(fmt.Sprintf("Deleting blob at %q", blobPath))
err := os.Remove(blobPath)
if err != nil && !os.IsNotExist(err) {
return err
} else {
return nil
}
}
func (ref ociReference) deleteReferenceFromIndex(referenceIndex int) error {
index, err := ref.getIndex()
if err != nil {
return err
}
index.Manifests = slices.Delete(index.Manifests, referenceIndex, referenceIndex+1)
return saveJSON(ref.indexPath(), index)
}
func saveJSON(path string, content any) error {
// If the file already exists, get its mode to preserve it
var mode fs.FileMode
existingfi, err := os.Stat(path)
if err != nil {
if !os.IsNotExist(err) {
return err
} else { // File does not exist, use default mode
mode = 0644
}
} else {
mode = existingfi.Mode()
}
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
defer file.Close()
return json.NewEncoder(file).Encode(content)
}
func (ref ociReference) getManifest(descriptor *imgspecv1.Descriptor, sharedBlobsDir string) (*imgspecv1.Manifest, error) {
manifestPath, err := ref.blobPath(descriptor.Digest, sharedBlobsDir)
if err != nil {
return nil, err
}
manifest, err := parseJSON[imgspecv1.Manifest](manifestPath)
if err != nil {
return nil, err
}
return manifest, nil
}

View File

@ -19,6 +19,7 @@ import (
digest "github.com/opencontainers/go-digest"
imgspec "github.com/opencontainers/image-spec/specs-go"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/exp/slices"
)
type ociImageDestination struct {
@ -84,7 +85,7 @@ func newImageDestination(sys *types.SystemContext, ref ociReference) (private.Im
// Per the OCI image specification, layouts MUST have a "blobs" subdirectory,
// but it MAY be empty (e.g. if we never end up calling PutBlob)
// https://github.com/opencontainers/image-spec/blame/7c889fafd04a893f5c5f50b7ab9963d5d64e5242/image-layout.md#L19
if err := ensureDirectoryExists(filepath.Join(d.ref.dir, "blobs")); err != nil {
if err := ensureDirectoryExists(filepath.Join(d.ref.dir, imgspecv1.ImageBlobsDir)); err != nil {
return nil, err
}
return d, nil
@ -271,8 +272,8 @@ func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) {
return
}
}
// It's a new entry to be added to the index.
d.index.Manifests = append(d.index.Manifests, *desc)
// It's a new entry to be added to the index. Use slices.Clone() to avoid a remote dependency on how d.index was created.
d.index.Manifests = append(slices.Clone(d.index.Manifests), *desc)
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
@ -283,7 +284,13 @@ func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) {
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *ociImageDestination) Commit(context.Context, types.UnparsedImage) error {
if err := os.WriteFile(d.ref.ociLayoutPath(), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0644); err != nil {
layoutBytes, err := json.Marshal(imgspecv1.ImageLayout{
Version: imgspecv1.ImageLayoutVersion,
})
if err != nil {
return err
}
if err := os.WriteFile(d.ref.ociLayoutPath(), layoutBytes, 0644); err != nil {
return err
}
indexJSON, err := json.Marshal(d.index)

View File

@ -60,7 +60,7 @@ func newImageSource(sys *types.SystemContext, ref ociReference) (private.ImageSo
client := &http.Client{}
client.Transport = tr
descriptor, err := ref.getManifestDescriptor()
descriptor, _, err := ref.getManifestDescriptor()
if err != nil {
return nil, err
}

View File

@ -160,48 +160,56 @@ func (ref ociReference) NewImage(ctx context.Context, sys *types.SystemContext)
// getIndex returns a pointer to the index references by this ociReference. If an error occurs opening an index nil is returned together
// with an error.
func (ref ociReference) getIndex() (*imgspecv1.Index, error) {
indexJSON, err := os.Open(ref.indexPath())
if err != nil {
return nil, err
}
defer indexJSON.Close()
index := &imgspecv1.Index{}
if err := json.NewDecoder(indexJSON).Decode(index); err != nil {
return nil, err
}
return index, nil
return parseIndex(ref.indexPath())
}
func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) {
func parseIndex(path string) (*imgspecv1.Index, error) {
return parseJSON[imgspecv1.Index](path)
}
func parseJSON[T any](path string) (*T, error) {
content, err := os.Open(path)
if err != nil {
return nil, err
}
defer content.Close()
obj := new(T)
if err := json.NewDecoder(content).Decode(obj); err != nil {
return nil, err
}
return obj, nil
}
func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, int, error) {
index, err := ref.getIndex()
if err != nil {
return imgspecv1.Descriptor{}, err
return imgspecv1.Descriptor{}, -1, err
}
if ref.image == "" {
// return manifest if only one image is in the oci directory
if len(index.Manifests) != 1 {
// ask user to choose image when more than one image in the oci directory
return imgspecv1.Descriptor{}, ErrMoreThanOneImage
return imgspecv1.Descriptor{}, -1, ErrMoreThanOneImage
}
return index.Manifests[0], nil
return index.Manifests[0], 0, nil
} else {
// if image specified, look through all manifests for a match
var unsupportedMIMETypes []string
for _, md := range index.Manifests {
for i, md := range index.Manifests {
if refName, ok := md.Annotations[imgspecv1.AnnotationRefName]; ok && refName == ref.image {
if md.MediaType == imgspecv1.MediaTypeImageManifest || md.MediaType == imgspecv1.MediaTypeImageIndex {
return md, nil
return md, i, nil
}
unsupportedMIMETypes = append(unsupportedMIMETypes, md.MediaType)
}
}
if len(unsupportedMIMETypes) != 0 {
return imgspecv1.Descriptor{}, fmt.Errorf("reference %q matches unsupported manifest MIME types %q", ref.image, unsupportedMIMETypes)
return imgspecv1.Descriptor{}, -1, fmt.Errorf("reference %q matches unsupported manifest MIME types %q", ref.image, unsupportedMIMETypes)
}
}
return imgspecv1.Descriptor{}, ImageNotFoundError{ref}
return imgspecv1.Descriptor{}, -1, ImageNotFoundError{ref}
}
// LoadManifestDescriptor loads the manifest descriptor to be used to retrieve the image name
@ -211,7 +219,8 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor,
if !ok {
return imgspecv1.Descriptor{}, errors.New("error typecasting, need type ociRef")
}
return ociRef.getManifestDescriptor()
md, _, err := ociRef.getManifestDescriptor()
return md, err
}
// NewImageSource returns a types.ImageSource for this reference.
@ -226,19 +235,14 @@ func (ref ociReference) NewImageDestination(ctx context.Context, sys *types.Syst
return newImageDestination(sys, ref)
}
// DeleteImage deletes the named image from the registry, if supported.
func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.New("Deleting images not implemented for oci: images")
}
// ociLayoutPath returns a path for the oci-layout within a directory using OCI conventions.
func (ref ociReference) ociLayoutPath() string {
return filepath.Join(ref.dir, "oci-layout")
return filepath.Join(ref.dir, imgspecv1.ImageLayoutFile)
}
// indexPath returns a path for the index.json within a directory using OCI conventions.
func (ref ociReference) indexPath() string {
return filepath.Join(ref.dir, "index.json")
return filepath.Join(ref.dir, imgspecv1.ImageIndexFile)
}
// blobPath returns a path for a blob within a directory using OCI image-layout conventions.
@ -246,9 +250,11 @@ func (ref ociReference) blobPath(digest digest.Digest, sharedBlobDir string) (st
if err := digest.Validate(); err != nil {
return "", fmt.Errorf("unexpected digest reference %s: %w", digest, err)
}
blobDir := filepath.Join(ref.dir, "blobs")
var blobDir string
if sharedBlobDir != "" {
blobDir = sharedBlobDir
} else {
blobDir = filepath.Join(ref.dir, imgspecv1.ImageBlobsDir)
}
return filepath.Join(blobDir, digest.Algorithm().String(), digest.Hex()), nil
}

View File

@ -10,15 +10,20 @@ import (
"github.com/opencontainers/go-digest"
)
// replacementAttempts is the number of blob replacement candidates returned by destructivelyPrioritizeReplacementCandidates,
// replacementAttempts is the number of blob replacement candidates with known location returned by destructivelyPrioritizeReplacementCandidates,
// and therefore ultimately by types.BlobInfoCache.CandidateLocations.
// This is a heuristic/guess, and could well use a different value.
const replacementAttempts = 5
// replacementUnknownLocationAttempts is the number of blob replacement candidates with unknown Location returned by destructivelyPrioritizeReplacementCandidates,
// and therefore ultimately by blobinfocache.BlobInfoCache2.CandidateLocations2.
// This is a heuristic/guess, and could well use a different value.
const replacementUnknownLocationAttempts = 2
// CandidateWithTime is the input to types.BICReplacementCandidate prioritization.
type CandidateWithTime struct {
Candidate blobinfocache.BICReplacementCandidate2 // The replacement candidate
LastSeen time.Time // Time the candidate was last known to exist (either read or written)
LastSeen time.Time // Time the candidate was last known to exist (either read or written) (not set for Candidate.UnknownLocation)
}
// candidateSortState is a local state implementing sort.Interface on candidates to prioritize,
@ -77,9 +82,22 @@ func (css *candidateSortState) Swap(i, j int) {
css.cs[i], css.cs[j] = css.cs[j], css.cs[i]
}
// destructivelyPrioritizeReplacementCandidatesWithMax is destructivelyPrioritizeReplacementCandidates with a parameter for the
// number of entries to limit, only to make testing simpler.
func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime, primaryDigest, uncompressedDigest digest.Digest, maxCandidates int) []blobinfocache.BICReplacementCandidate2 {
func min(a, b int) int {
if a < b {
return a
}
return b
}
// destructivelyPrioritizeReplacementCandidatesWithMax is destructivelyPrioritizeReplacementCandidates with parameters for the
// number of entries to limit for known and unknown location separately, only to make testing simpler.
// TODO: following function is not destructive any more in the nature instead priortized result is actually copies of the original
// candidate set, so In future we might wanna re-name this public API and remove the destructive prefix.
func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime, primaryDigest, uncompressedDigest digest.Digest, totalLimit int, noLocationLimit int) []blobinfocache.BICReplacementCandidate2 {
// split unknown candidates and known candidates
// and limit them seperately.
var knownLocationCandidates []CandidateWithTime
var unknownLocationCandidates []CandidateWithTime
// We don't need to use sort.Stable() because nanosecond timestamps are (presumably?) unique, so no two elements should
// compare equal.
// FIXME: Use slices.SortFunc after we update to Go 1.20 (Go 1.21?) and Time.Compare and cmp.Compare are available.
@ -88,24 +106,34 @@ func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime,
primaryDigest: primaryDigest,
uncompressedDigest: uncompressedDigest,
})
resLength := len(cs)
if resLength > maxCandidates {
resLength = maxCandidates
for _, candidate := range cs {
if candidate.Candidate.UnknownLocation {
unknownLocationCandidates = append(unknownLocationCandidates, candidate)
} else {
knownLocationCandidates = append(knownLocationCandidates, candidate)
}
}
res := make([]blobinfocache.BICReplacementCandidate2, resLength)
for i := range res {
res[i] = cs[i].Candidate
knownLocationCandidatesUsed := min(len(knownLocationCandidates), totalLimit)
remainingCapacity := totalLimit - knownLocationCandidatesUsed
unknownLocationCandidatesUsed := min(noLocationLimit, min(remainingCapacity, len(unknownLocationCandidates)))
res := make([]blobinfocache.BICReplacementCandidate2, knownLocationCandidatesUsed)
for i := 0; i < knownLocationCandidatesUsed; i++ {
res[i] = knownLocationCandidates[i].Candidate
}
// If candidates with unknown location are found, lets add them to final list
for i := 0; i < unknownLocationCandidatesUsed; i++ {
res = append(res, unknownLocationCandidates[i].Candidate)
}
return res
}
// DestructivelyPrioritizeReplacementCandidates consumes AND DESTROYS an array of possible replacement candidates with their last known existence times,
// the primary digest the user actually asked for, and the corresponding uncompressed digest (if known, possibly equal to the primary digest),
// and returns an appropriately prioritized and/or trimmed result suitable for a return value from types.BlobInfoCache.CandidateLocations.
// the primary digest the user actually asked for, the corresponding uncompressed digest (if known, possibly equal to the primary digest) returns an
// appropriately prioritized and/or trimmed result suitable for a return value from types.BlobInfoCache.CandidateLocations.
//
// WARNING: The array of candidates is destructively modified. (The implementation of this function could of course
// make a copy, but all CandidateLocations implementations build the slice of candidates only for the single purpose of calling this function anyway.)
func DestructivelyPrioritizeReplacementCandidates(cs []CandidateWithTime, primaryDigest, uncompressedDigest digest.Digest) []blobinfocache.BICReplacementCandidate2 {
return destructivelyPrioritizeReplacementCandidatesWithMax(cs, primaryDigest, uncompressedDigest, replacementAttempts)
return destructivelyPrioritizeReplacementCandidatesWithMax(cs, primaryDigest, uncompressedDigest, replacementAttempts, replacementUnknownLocationAttempts)
}

View File

@ -133,24 +133,39 @@ func (mem *cache) RecordDigestCompressorName(blobDigest digest.Digest, compresso
mem.compressors[blobDigest] = compressorName
}
// appendReplacementCandidates creates prioritize.CandidateWithTime values for (transport, scope, digest), and returns the result of appending them to candidates.
func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, requireCompressionInfo bool) []prioritize.CandidateWithTime {
// appendReplacementCandidates creates prioritize.CandidateWithTime values for digest in memory
// with corresponding compression info from mem.compressors, and returns the result of appending
// them to candidates. v2Output allows including candidates with unknown location, and filters out
// candidates with unknown compression.
func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, v2Output bool) []prioritize.CandidateWithTime {
compressorName := blobinfocache.UnknownCompression
if v, ok := mem.compressors[digest]; ok {
compressorName = v
}
if compressorName == blobinfocache.UnknownCompression && v2Output {
return candidates
}
locations := mem.knownLocations[locationKey{transport: transport.Name(), scope: scope, blobDigest: digest}] // nil if not present
for l, t := range locations {
compressorName, compressorKnown := mem.compressors[digest]
if !compressorKnown {
if requireCompressionInfo {
continue
}
compressorName = blobinfocache.UnknownCompression
if len(locations) > 0 {
for l, t := range locations {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressorName: compressorName,
Location: l,
},
LastSeen: t,
})
}
} else if v2Output {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressorName: compressorName,
Location: l,
Digest: digest,
CompressorName: compressorName,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: t,
LastSeen: time.Time{},
})
}
return candidates
@ -166,7 +181,7 @@ func (mem *cache) CandidateLocations(transport types.ImageTransport, scope types
return blobinfocache.CandidateLocationsFromV2(mem.candidateLocations(transport, scope, primaryDigest, canSubstitute, false))
}
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations that could possibly be reused
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations (if known) that could possibly be reused
// within the specified (transport scope) (if they still exist, which is not guaranteed).
//
// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute,
@ -176,23 +191,24 @@ func (mem *cache) CandidateLocations2(transport types.ImageTransport, scope type
return mem.candidateLocations(transport, scope, primaryDigest, canSubstitute, true)
}
func (mem *cache) candidateLocations(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute, requireCompressionInfo bool) []blobinfocache.BICReplacementCandidate2 {
func (mem *cache) candidateLocations(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute, v2Output bool) []blobinfocache.BICReplacementCandidate2 {
mem.mutex.Lock()
defer mem.mutex.Unlock()
res := []prioritize.CandidateWithTime{}
res = mem.appendReplacementCandidates(res, transport, scope, primaryDigest, requireCompressionInfo)
res = mem.appendReplacementCandidates(res, transport, scope, primaryDigest, v2Output)
var uncompressedDigest digest.Digest // = ""
if canSubstitute {
if uncompressedDigest = mem.uncompressedDigestLocked(primaryDigest); uncompressedDigest != "" {
if otherDigests, ok := mem.digestsByUncompressed[uncompressedDigest]; ok {
otherDigests := mem.digestsByUncompressed[uncompressedDigest] // nil if not present in the map
if otherDigests != nil {
for _, d := range otherDigests.Values() {
if d != primaryDigest && d != uncompressedDigest {
res = mem.appendReplacementCandidates(res, transport, scope, d, requireCompressionInfo)
res = mem.appendReplacementCandidates(res, transport, scope, d, v2Output)
}
}
}
if uncompressedDigest != primaryDigest {
res = mem.appendReplacementCandidates(res, transport, scope, uncompressedDigest, requireCompressionInfo)
res = mem.appendReplacementCandidates(res, transport, scope, uncompressedDigest, v2Output)
}
}
}

View File

@ -57,7 +57,7 @@ type cache struct {
// The database/sql package says “It is rarely necessary to close a DB.”, and steers towards a long-term *sql.DB connection pool.
// Thats probably very applicable for database-backed services, where the database is the primary data store. Thats not necessarily
// the case for callers of c/image, where image operations might be a small proportion of hte total runtime, and the cache is fairly
// the case for callers of c/image, where image operations might be a small proportion of the total runtime, and the cache is fairly
// incidental even to the image operations. Its also hard for us to use that model, because the public BlobInfoCache object doesnt have
// a Close method, so creating a lot of single-use caches could leak data.
//
@ -117,7 +117,7 @@ func (sqc *cache) Open() {
if sqc.refCount == 0 {
db, err := rawOpen(sqc.path)
if err != nil {
logrus.Warnf("Error opening (previously-succesfully-opened) blob info cache at %q: %v", sqc.path, err)
logrus.Warnf("Error opening (previously-successfully-opened) blob info cache at %q: %v", sqc.path, err)
db = nil // But still increase sqc.refCount, because a .Close() will happen
}
sqc.db = db
@ -171,7 +171,7 @@ func transaction[T any](sqc *cache, fn func(tx *sql.Tx) (T, error)) (T, error) {
// dbTransaction calls fn within a read-write transaction in db.
func dbTransaction[T any](db *sql.DB, fn func(tx *sql.Tx) (T, error)) (T, error) {
// Ideally we should be able to distinguish between read-only and read-write transctions, see the _txlock=exclusive dicussion.
// Ideally we should be able to distinguish between read-only and read-write transactions, see the _txlock=exclusive dicussion.
var zeroRes T // A zero value of T
@ -249,7 +249,7 @@ func ensureDBHasCurrentSchema(db *sql.DB) error {
// * Joins (the two that exist in appendReplacementCandidates) are based on the text representation of digests.
//
// Using integer primary keys might make the joins themselves a bit more efficient, but then we would need to involve an extra
// join to translate from/to the user-provided digests anyway. If anything, that extra join (potentialy more btree lookups)
// join to translate from/to the user-provided digests anyway. If anything, that extra join (potentially more btree lookups)
// is probably costlier than comparing a few more bytes of data.
//
// Perhaps more importantly, storing digest texts directly makes the database dumps much easier to read for humans without
@ -427,11 +427,13 @@ func (sqc *cache) RecordDigestCompressorName(anyDigest digest.Digest, compressor
}) // FIXME? Log error (but throttle the log volume on repeated accesses)?
}
// appendReplacementCandidates creates prioritize.CandidateWithTime values for (transport, scope, digest), and returns the result of appending them to candidates.
func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, tx *sql.Tx, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, requireCompressionInfo bool) ([]prioritize.CandidateWithTime, error) {
// appendReplacementCandidates creates prioritize.CandidateWithTime values for (transport, scope, digest),
// and returns the result of appending them to candidates. v2Output allows including candidates with unknown
// location, and filters out candidates with unknown compression.
func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, tx *sql.Tx, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, v2Output bool) ([]prioritize.CandidateWithTime, error) {
var rows *sql.Rows
var err error
if requireCompressionInfo {
if v2Output {
rows, err = tx.Query("SELECT location, time, compressor FROM KnownLocations JOIN DigestCompressors "+
"ON KnownLocations.digest = DigestCompressors.digest "+
"WHERE transport = ? AND scope = ? AND KnownLocations.digest = ?",
@ -448,6 +450,7 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
}
defer rows.Close()
res := []prioritize.CandidateWithTime{}
for rows.Next() {
var location string
var time time.Time
@ -455,7 +458,7 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if err := rows.Scan(&location, &time, &compressorName); err != nil {
return nil, fmt.Errorf("scanning candidate: %w", err)
}
candidates = append(candidates, prioritize.CandidateWithTime{
res = append(res, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressorName: compressorName,
@ -467,10 +470,29 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterating through locations: %w", err)
}
if len(res) == 0 && v2Output {
compressor, found, err := querySingleValue[string](tx, "SELECT compressor FROM DigestCompressors WHERE digest = ?", digest.String())
if err != nil {
return nil, fmt.Errorf("scanning compressorName: %w", err)
}
if found {
res = append(res, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressorName: compressor,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: time.Time{},
})
}
}
candidates = append(candidates, res...)
return candidates, nil
}
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations (if known)
// that could possibly be reused within the specified (transport scope) (if they still
// exist, which is not guaranteed).
//
@ -483,11 +505,11 @@ func (sqc *cache) CandidateLocations2(transport types.ImageTransport, scope type
return sqc.candidateLocations(transport, scope, digest, canSubstitute, true)
}
func (sqc *cache) candidateLocations(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute, requireCompressionInfo bool) []blobinfocache.BICReplacementCandidate2 {
func (sqc *cache) candidateLocations(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute, v2Output bool) []blobinfocache.BICReplacementCandidate2 {
var uncompressedDigest digest.Digest // = ""
res, err := transaction(sqc, func(tx *sql.Tx) ([]prioritize.CandidateWithTime, error) {
res := []prioritize.CandidateWithTime{}
res, err := sqc.appendReplacementCandidates(res, tx, transport, scope, primaryDigest, requireCompressionInfo)
res, err := sqc.appendReplacementCandidates(res, tx, transport, scope, primaryDigest, v2Output)
if err != nil {
return nil, err
}
@ -516,7 +538,7 @@ func (sqc *cache) candidateLocations(transport types.ImageTransport, scope types
return nil, err
}
if otherDigest != primaryDigest && otherDigest != uncompressedDigest {
res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, otherDigest, requireCompressionInfo)
res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, otherDigest, v2Output)
if err != nil {
return nil, err
}
@ -527,7 +549,7 @@ func (sqc *cache) candidateLocations(transport types.ImageTransport, scope types
}
if uncompressedDigest != primaryDigest {
res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, uncompressedDigest, requireCompressionInfo)
res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, uncompressedDigest, v2Output)
if err != nil {
return nil, err
}

View File

@ -11,6 +11,7 @@ import (
"github.com/containers/image/v5/types"
"github.com/manifoldco/promptui"
"github.com/opencontainers/go-digest"
"golang.org/x/exp/slices"
"golang.org/x/term"
)
@ -169,7 +170,7 @@ func (r *Resolved) Description() string {
// pull errors must equal the amount of pull candidates.
func (r *Resolved) FormatPullErrors(pullErrors []error) error {
if len(pullErrors) > 0 && len(pullErrors) != len(r.PullCandidates) {
pullErrors = append(pullErrors,
pullErrors = append(slices.Clone(pullErrors),
fmt.Errorf("internal error: expected %d instead of %d errors for %d pull candidates",
len(r.PullCandidates), len(pullErrors), len(r.PullCandidates)))
}

View File

@ -66,7 +66,7 @@ func SetupCertificates(dir string, tlsc *tls.Config) error {
if err != nil {
return err
}
tlsc.Certificates = append(tlsc.Certificates, cert)
tlsc.Certificates = append(slices.Clone(tlsc.Certificates), cert)
}
if strings.HasSuffix(f.Name(), ".key") {
keyName := f.Name()

View File

@ -10,6 +10,7 @@ import (
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
digest "github.com/opencontainers/go-digest"
@ -283,3 +284,29 @@ func (s storageReference) NewImageSource(ctx context.Context, sys *types.SystemC
func (s storageReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(sys, s)
}
// ResolveReference finds the underlying storage image for a storage.Transport reference.
// It returns that image, and an updated reference which can be used to refer back to the _same_
// image again.
//
// This matters if the input reference contains a tagged name; the destination of the tag can
// move in local storage. The updated reference returned by this function contains the resolved
// image ID, so later uses of that updated reference will either continue to refer to the same
// image, or fail.
//
// Note that it _is_ possible for the later uses to fail, either because the image was removed
// completely, or because the name used in the reference was untaged (even if the underlying image
// ID still exists in local storage).
func ResolveReference(ref types.ImageReference) (types.ImageReference, *storage.Image, error) {
sref, ok := ref.(*storageReference)
if !ok {
return nil, nil, fmt.Errorf("trying to resolve a non-%s: reference %q", Transport.Name(),
transports.ImageName(ref))
}
clone := *sref // A shallow copy we can update
img, err := clone.resolveImage(nil)
if err != nil {
return nil, nil, err
}
return clone, img, nil
}

View File

@ -48,9 +48,24 @@ type StoreTransport interface {
GetStoreIfSet() storage.Store
// GetImage retrieves the image from the transport's store that's named
// by the reference.
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Use storage.ResolveReference instead.
GetImage(types.ImageReference) (*storage.Image, error)
// GetStoreImage retrieves the image from a specified store that's named
// by the reference.
//
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Also, a StoreTransport reference already contains a store, so providing another one is redundant.
//
// Use storage.ResolveReference instead.
GetStoreImage(storage.Store, types.ImageReference) (*storage.Image, error)
// ParseStoreReference parses a reference, overriding any store
// specification that it may contain.
@ -290,6 +305,14 @@ func (s *storageTransport) ParseReference(reference string) (types.ImageReferenc
return s.ParseStoreReference(store, reference)
}
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Also, a StoreTransport reference already contains a store, so providing another one is redundant.
//
// Use storage.ResolveReference instead.
func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageReference) (*storage.Image, error) {
dref := ref.DockerReference()
if dref != nil {
@ -306,6 +329,12 @@ func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageRefe
return nil, storage.ErrImageUnknown
}
// Deprecated: Surprisingly, with a StoreTransport reference which contains an ID,
// this ignores that ID; and repeated calls of GetStoreImage with the same named reference
// can return different images, with no way for the caller to "freeze" the storage.Image identity
// without discarding the name entirely.
//
// Use storage.ResolveReference instead.
func (s *storageTransport) GetImage(ref types.ImageReference) (*storage.Image, error) {
store, err := s.GetStore()
if err != nil {

View File

@ -445,7 +445,7 @@ type ImageCloser interface {
Close() error
}
// ManifestUpdateOptions is a way to pass named optional arguments to Image.UpdatedManifest
// ManifestUpdateOptions is a way to pass named optional arguments to Image.UpdatedImage
type ManifestUpdateOptions struct {
LayerInfos []BlobInfo // Complete BlobInfos (size+digest+urls+annotations) which should replace the originals, in order (the root layer first, and then successive layered layers). BlobInfos' MediaType fields are ignored.
EmbeddedDockerReference reference.Named
@ -457,7 +457,7 @@ type ManifestUpdateOptions struct {
// ManifestUpdateInformation is a component of ManifestUpdateOptions, named here
// only to make writing struct literals possible.
type ManifestUpdateInformation struct {
Destination ImageDestination // and yes, UpdatedManifest may write to Destination (see the schema2 → schema1 conversion logic in image/docker_schema2.go)
Destination ImageDestination // and yes, UpdatedImage may write to Destination (see the schema2 → schema1 conversion logic in image/docker_schema2.go)
LayerInfos []BlobInfo // Complete BlobInfos (size+digest) which have been uploaded, in order (the root layer first, and then successive layered layers)
LayerDiffIDs []digest.Digest // Digest values for the _uncompressed_ contents of the blobs which have been uploaded, in the same order.
}

View File

@ -8,10 +8,10 @@ const (
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 28
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 0
VersionPatch = 1
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = ""
VersionDev = "-dev"
)
// Version is the specification version that the package types support.

View File

@ -28,6 +28,7 @@ vendor:
go mod tidy
test:
go clean -testcache
go test ./... -test.v
generate-protobuf:

View File

@ -41,7 +41,11 @@ func NewKeyWrapper() keywrap.KeyWrapper {
// WrapKeys wraps the session key for recpients and encrypts the optsData, which
// describe the symmetric key used for encrypting the layer
func (kw *pkcs11KeyWrapper) WrapKeys(ec *config.EncryptConfig, optsData []byte) ([]byte, error) {
pkcs11Recipients, err := addPubKeys(&ec.DecryptConfig, append(ec.Parameters["pkcs11-pubkeys"], ec.Parameters["pkcs11-yamls"]...))
// append({}, ...) allocates a fresh backing array, and that's necessary to guarantee concurrent calls to WrapKeys (as in c/image/copy.Image)
// can't race writing to the same backing array.
pubKeys := append([][]byte{}, ec.Parameters["pkcs11-pubkeys"]...) // In Go 1.21, slices.Clone(ec.Parameters["pkcs11-pubkeys"])
pubKeys = append(pubKeys, ec.Parameters["pkcs11-yamls"]...)
pkcs11Recipients, err := addPubKeys(&ec.DecryptConfig, pubKeys)
if err != nil {
return nil, err
}

View File

@ -9,8 +9,12 @@ const (
MediaTypeLayerZstdEnc = "application/vnd.oci.image.layer.v1.tar+zstd+encrypted"
// MediaTypeLayerNonDistributableEnc is MIME type used for non distributable encrypted layers.
MediaTypeLayerNonDistributableEnc = "application/vnd.oci.image.layer.nondistributable.v1.tar+encrypted"
// MediaTypeLayerGzipEnc is MIME type used for non distributable encrypted gzip-compressed layers.
// MediaTypeLayerNonDistributableGzipEnc is MIME type used for non distributable encrypted gzip-compressed layers.
MediaTypeLayerNonDistributableGzipEnc = "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip+encrypted"
// MediaTypeLayerZstdEnc is MIME type used for non distributable encrypted zstd-compressed layers.
MediaTypeLayerNonDistributableZsdtEnc = "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd+encrypted"
// MediaTypeLayerNonDistributableZstdEnc is MIME type used for non distributable encrypted zstd-compressed layers.
MediaTypeLayerNonDistributableZstdEnc = "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd+encrypted"
// MediaTypeLayerNonDistributableZsdtEnc is MIME type used for non distributable encrypted zstd-compressed layers.
//
// Deprecated: Use [MediaTypeLayerNonDistributableZstdEnc].
MediaTypeLayerNonDistributableZsdtEnc = MediaTypeLayerNonDistributableZstdEnc
)

View File

@ -17,13 +17,13 @@ env:
####
#### Cache-image names to test with (double-quotes around names are critical)
###
FEDORA_NAME: "fedora-38"
FEDORA_NAME: "fedora-3"
DEBIAN_NAME: "debian-13"
# GCE project where images live
IMAGE_PROJECT: "libpod-218412"
# VM Image built in containers/automation_images
IMAGE_SUFFIX: "c20230816t191118z-f38f37d13"
IMAGE_SUFFIX: "c20231004t194547z-f39f38d13"
FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}"
@ -113,8 +113,6 @@ debian_testing_task: &debian_testing
TEST_DRIVER: "fuse-overlay-whiteout"
- env:
TEST_DRIVER: "btrfs"
- env:
TEST_DRIVER: "zfs"
lint_task:

View File

@ -1245,8 +1245,8 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
if parentLayer != nil {
parent = parentLayer.ID
}
var parentMappings, templateIDMappings, oldMappings *idtools.IDMappings
var (
templateIDMappings *idtools.IDMappings
templateMetadata string
templateCompressedDigest digest.Digest
templateCompressedSize int64
@ -1274,11 +1274,6 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
} else {
templateIDMappings = &idtools.IDMappings{}
}
if parentLayer != nil {
parentMappings = idtools.NewIDMappingsFromMaps(parentLayer.UIDMap, parentLayer.GIDMap)
} else {
parentMappings = &idtools.IDMappings{}
}
if mountLabel != "" {
selinux.ReserveLabel(mountLabel)
}
@ -1353,6 +1348,12 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
IDMappings: idMappings,
}
var parentMappings, oldMappings *idtools.IDMappings
if parentLayer != nil {
parentMappings = idtools.NewIDMappingsFromMaps(parentLayer.UIDMap, parentLayer.GIDMap)
} else {
parentMappings = &idtools.IDMappings{}
}
if moreOptions.TemplateLayer != "" {
if err = r.driver.CreateFromTemplate(id, moreOptions.TemplateLayer, templateIDMappings, parent, parentMappings, &opts, writeable); err != nil {
cleanupFailureContext = fmt.Sprintf("creating a layer from template layer %q", moreOptions.TemplateLayer)
@ -1371,10 +1372,13 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
return nil, -1, fmt.Errorf("creating read-only layer with ID %q: %w", id, err)
}
}
oldMappings = parentMappings
if parentLayer != nil {
oldMappings = parentMappings
}
}
if !reflect.DeepEqual(oldMappings.UIDs(), idMappings.UIDs()) || !reflect.DeepEqual(oldMappings.GIDs(), idMappings.GIDs()) {
if oldMappings != nil &&
(!reflect.DeepEqual(oldMappings.UIDs(), idMappings.UIDs()) || !reflect.DeepEqual(oldMappings.GIDs(), idMappings.GIDs())) {
if err = r.driver.UpdateLayerIDMap(id, oldMappings, idMappings, mountLabel); err != nil {
cleanupFailureContext = "in UpdateLayerIDMap"
return nil, -1, err

View File

@ -955,14 +955,8 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
if options.ForceMask != nil {
// if ForceMask is in place, make sure lchown is disabled.
doChown = false
uid, gid, mode, err := GetFileOwner(dest)
if err == nil {
value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode)
if err := system.Lsetxattr(dest, idtools.ContainersOverrideXattr, []byte(value), 0); err != nil {
return err
}
}
}
var rootHdr *tar.Header
// Iterate through the files in the archive.
loop:
@ -1007,6 +1001,9 @@ loop:
if err != nil {
return err
}
if rel == "." {
rootHdr = hdr
}
if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
}
@ -1080,6 +1077,14 @@ loop:
return err
}
}
if options.ForceMask != nil && rootHdr != nil {
value := fmt.Sprintf("%d:%d:0%o", rootHdr.Uid, rootHdr.Gid, rootHdr.Mode)
if err := system.Lsetxattr(dest, idtools.ContainersOverrideXattr, []byte(value), 0); err != nil {
return err
}
}
return nil
}

View File

@ -8,7 +8,7 @@ import (
"crypto/rsa"
"errors"
"fmt"
"io/ioutil"
"io"
"net/http"
"sync"
"time"
@ -159,7 +159,7 @@ func (r *RemoteKeySet) verify(ctx context.Context, jws *jose.JSONWebSignature) (
// https://openid.net/specs/openid-connect-core-1_0.html#RotateSigKeys
keys, err := r.keysFromRemote(ctx)
if err != nil {
return nil, fmt.Errorf("fetching keys %v", err)
return nil, fmt.Errorf("fetching keys %w", err)
}
for _, key := range keys {
@ -228,11 +228,11 @@ func (r *RemoteKeySet) updateKeys() ([]jose.JSONWebKey, error) {
resp, err := doRequest(r.ctx, req)
if err != nil {
return nil, fmt.Errorf("oidc: get keys failed %v", err)
return nil, fmt.Errorf("oidc: get keys failed %w", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("unable to read response body: %v", err)
}

View File

@ -10,7 +10,7 @@ import (
"errors"
"fmt"
"hash"
"io/ioutil"
"io"
"mime"
"net/http"
"strings"
@ -211,7 +211,7 @@ func NewProvider(ctx context.Context, issuer string) (*Provider, error) {
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("unable to read response body: %v", err)
}
@ -332,7 +332,7 @@ func (p *Provider) UserInfo(ctx context.Context, tokenSource oauth2.TokenSource)
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}

View File

@ -7,7 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"io"
"net/http"
"strings"
"time"
@ -182,7 +182,7 @@ func resolveDistributedClaim(ctx context.Context, verifier *IDTokenVerifier, src
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("unable to read response body: %v", err)
}

View File

@ -197,12 +197,13 @@ encodeLoop:
// Set m to a match at offset if it looks like that will improve compression.
improve := func(m *match, offset int32, s int32, first uint32, rep int32) {
if s-offset >= e.maxMatchOff || load3232(src, offset) != first {
delta := s - offset
if delta >= e.maxMatchOff || delta <= 0 || load3232(src, offset) != first {
return
}
if debugAsserts {
if offset <= 0 {
panic(offset)
if offset >= s {
panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff))
}
if !bytes.Equal(src[s:s+4], src[offset:offset+4]) {
panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first))
@ -343,8 +344,8 @@ encodeLoop:
if best.rep > 0 {
var seq seq
seq.matchLen = uint32(best.length - zstdMinMatch)
if debugAsserts && s <= nextEmit {
panic("s <= nextEmit")
if debugAsserts && s < nextEmit {
panic("s < nextEmit")
}
addLiterals(&seq, best.s)

View File

@ -26,7 +26,7 @@ import (
"errors"
"fmt"
"github.com/theupdateframework/go-tuf/encrypted"
"github.com/secure-systems-lab/go-securesystemslib/encrypted"
)
const (

View File

@ -27,7 +27,7 @@ import (
const CosignSignatureType = "cosign container image signature"
// SimpleContainerImage describes the structure of a basic container image signature payload, as defined at:
// https://github.com/containers/image/blob/master/docs/containers-signature.5.md#json-data-format
// https://github.com/containers/image/blob/main/docs/containers-signature.5.md#json-data-format
type SimpleContainerImage struct {
Critical Critical `json:"critical"` // Critical data critical to correctly evaluating the validity of the signature
Optional map[string]interface{} `json:"optional"` // Optional optional metadata about the image

View File

@ -8,6 +8,7 @@
package sif
import (
"encoding"
"encoding/binary"
"errors"
"fmt"
@ -321,51 +322,6 @@ func CreateContainerAtPath(path string, opts ...CreateOpt) (*FileImage, error) {
return f, nil
}
func zeroData(fimg *FileImage, descr *rawDescriptor) error {
// first, move to data object offset
if _, err := fimg.rw.Seek(descr.Offset, io.SeekStart); err != nil {
return err
}
var zero [4096]byte
n := descr.Size
upbound := int64(4096)
for {
if n < 4096 {
upbound = n
}
if _, err := fimg.rw.Write(zero[:upbound]); err != nil {
return err
}
n -= 4096
if n <= 0 {
break
}
}
return nil
}
func resetDescriptor(fimg *FileImage, index int) error {
// If we remove the primary partition, set the global header Arch field to HdrArchUnknown
// to indicate that the SIF file doesn't include a primary partition and no dependency
// on any architecture exists.
if fimg.rds[index].isPartitionOfType(PartPrimSys) {
fimg.h.Arch = hdrArchUnknown
}
offset := fimg.h.DescriptorsOffset + int64(index)*int64(binary.Size(fimg.rds[0]))
// first, move to descriptor offset
if _, err := fimg.rw.Seek(offset, io.SeekStart); err != nil {
return err
}
var emptyDesc rawDescriptor
return binary.Write(fimg.rw, binary.LittleEndian, emptyDesc)
}
// addOpts accumulates object add options.
type addOpts struct {
t time.Time
@ -447,6 +403,26 @@ func (f *FileImage) isLast(d *rawDescriptor) bool {
return isLast
}
// zeroReader is an io.Reader that returns a stream of zero-bytes.
type zeroReader struct{}
func (zeroReader) Read(b []byte) (int, error) {
for i := range b {
b[i] = 0
}
return len(b), nil
}
// zero overwrites the data object described by d with a stream of zero bytes.
func (f *FileImage) zero(d *rawDescriptor) error {
if _, err := f.rw.Seek(d.Offset, io.SeekStart); err != nil {
return err
}
_, err := io.CopyN(f.rw, zeroReader{}, d.Size)
return err
}
// truncateAt truncates f at the start of the padded data object described by d.
func (f *FileImage) truncateAt(d *rawDescriptor) error {
start := d.Offset + d.Size - d.SizeWithPadding
@ -530,7 +506,7 @@ func (f *FileImage) DeleteObject(id uint32, opts ...DeleteOpt) error {
}
if do.zero {
if err := zeroData(f, d); err != nil {
if err := f.zero(d); err != nil {
return fmt.Errorf("%w", err)
}
}
@ -546,15 +522,17 @@ func (f *FileImage) DeleteObject(id uint32, opts ...DeleteOpt) error {
f.h.DescriptorsFree++
f.h.ModifiedAt = do.t.Unix()
index := 0
for i, od := range f.rds {
if od.ID == id {
index = i
break
}
// If we remove the primary partition, set the global header Arch field to HdrArchUnknown
// to indicate that the SIF file doesn't include a primary partition and no dependency
// on any architecture exists.
if d.isPartitionOfType(PartPrimSys) {
f.h.Arch = hdrArchUnknown
}
if err := resetDescriptor(f, index); err != nil {
// Reset rawDescripter with empty struct
*d = rawDescriptor{}
if err := f.writeDescriptors(); err != nil {
return fmt.Errorf("%w", err)
}
@ -676,3 +654,45 @@ func (f *FileImage) SetPrimPart(id uint32, opts ...SetOpt) error {
return nil
}
// SetMetadata sets the metadata of the data object with id to md, according to opts.
//
// By default, the image/object modification times are set to the current time for
// non-deterministic images, and unset otherwise. To override this, consider using
// OptSetDeterministic or OptSetWithTime.
func (f *FileImage) SetMetadata(id uint32, md encoding.BinaryMarshaler, opts ...SetOpt) error {
so := setOpts{}
if !f.isDeterministic() {
so.t = time.Now()
}
for _, opt := range opts {
if err := opt(&so); err != nil {
return fmt.Errorf("%w", err)
}
}
rd, err := f.getDescriptor(WithID(id))
if err != nil {
return fmt.Errorf("%w", err)
}
if err := rd.setExtra(md); err != nil {
return fmt.Errorf("%w", err)
}
rd.ModifiedAt = so.t.Unix()
if err := f.writeDescriptors(); err != nil {
return fmt.Errorf("%w", err)
}
f.h.ModifiedAt = so.t.Unix()
if err := f.writeHeader(); err != nil {
return fmt.Errorf("%w", err)
}
return nil
}

View File

@ -1,27 +0,0 @@
Copyright (c) 2014-2020 Prime Directive, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Prime Directive, Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,226 +0,0 @@
// Package encrypted provides a simple, secure system for encrypting data
// symmetrically with a passphrase.
//
// It uses scrypt derive a key from the passphrase and the NaCl secret box
// cipher for authenticated encryption.
package encrypted
import (
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"io"
"golang.org/x/crypto/nacl/secretbox"
"golang.org/x/crypto/scrypt"
)
const saltSize = 32
const (
boxKeySize = 32
boxNonceSize = 24
)
const (
// N parameter was chosen to be ~100ms of work using the default implementation
// on the 2.3GHz Core i7 Haswell processor in a late-2013 Apple Retina Macbook
// Pro (it takes ~113ms).
scryptN = 32768
scryptR = 8
scryptP = 1
)
const (
nameScrypt = "scrypt"
nameSecretBox = "nacl/secretbox"
)
type data struct {
KDF scryptKDF `json:"kdf"`
Cipher secretBoxCipher `json:"cipher"`
Ciphertext []byte `json:"ciphertext"`
}
type scryptParams struct {
N int `json:"N"`
R int `json:"r"`
P int `json:"p"`
}
func newScryptKDF() (scryptKDF, error) {
salt := make([]byte, saltSize)
if err := fillRandom(salt); err != nil {
return scryptKDF{}, err
}
return scryptKDF{
Name: nameScrypt,
Params: scryptParams{
N: scryptN,
R: scryptR,
P: scryptP,
},
Salt: salt,
}, nil
}
type scryptKDF struct {
Name string `json:"name"`
Params scryptParams `json:"params"`
Salt []byte `json:"salt"`
}
func (s *scryptKDF) Key(passphrase []byte) ([]byte, error) {
return scrypt.Key(passphrase, s.Salt, s.Params.N, s.Params.R, s.Params.P, boxKeySize)
}
// CheckParams checks that the encoded KDF parameters are what we expect them to
// be. If we do not do this, an attacker could cause a DoS by tampering with
// them.
func (s *scryptKDF) CheckParams() error {
if s.Params.N != scryptN || s.Params.R != scryptR || s.Params.P != scryptP {
return errors.New("encrypted: unexpected kdf parameters")
}
return nil
}
func newSecretBoxCipher() (secretBoxCipher, error) {
nonce := make([]byte, boxNonceSize)
if err := fillRandom(nonce); err != nil {
return secretBoxCipher{}, err
}
return secretBoxCipher{
Name: nameSecretBox,
Nonce: nonce,
}, nil
}
type secretBoxCipher struct {
Name string `json:"name"`
Nonce []byte `json:"nonce"`
encrypted bool
}
func (s *secretBoxCipher) Encrypt(plaintext, key []byte) []byte {
var keyBytes [boxKeySize]byte
var nonceBytes [boxNonceSize]byte
if len(key) != len(keyBytes) {
panic("incorrect key size")
}
if len(s.Nonce) != len(nonceBytes) {
panic("incorrect nonce size")
}
copy(keyBytes[:], key)
copy(nonceBytes[:], s.Nonce)
// ensure that we don't re-use nonces
if s.encrypted {
panic("Encrypt must only be called once for each cipher instance")
}
s.encrypted = true
return secretbox.Seal(nil, plaintext, &nonceBytes, &keyBytes)
}
func (s *secretBoxCipher) Decrypt(ciphertext, key []byte) ([]byte, error) {
var keyBytes [boxKeySize]byte
var nonceBytes [boxNonceSize]byte
if len(key) != len(keyBytes) {
panic("incorrect key size")
}
if len(s.Nonce) != len(nonceBytes) {
// return an error instead of panicking since the nonce is user input
return nil, errors.New("encrypted: incorrect nonce size")
}
copy(keyBytes[:], key)
copy(nonceBytes[:], s.Nonce)
res, ok := secretbox.Open(nil, ciphertext, &nonceBytes, &keyBytes)
if !ok {
return nil, errors.New("encrypted: decryption failed")
}
return res, nil
}
// Encrypt takes a passphrase and plaintext, and returns a JSON object
// containing ciphertext and the details necessary to decrypt it.
func Encrypt(plaintext, passphrase []byte) ([]byte, error) {
k, err := newScryptKDF()
if err != nil {
return nil, err
}
key, err := k.Key(passphrase)
if err != nil {
return nil, err
}
c, err := newSecretBoxCipher()
if err != nil {
return nil, err
}
data := &data{
KDF: k,
Cipher: c,
}
data.Ciphertext = c.Encrypt(plaintext, key)
return json.Marshal(data)
}
// Marshal encrypts the JSON encoding of v using passphrase.
func Marshal(v interface{}, passphrase []byte) ([]byte, error) {
data, err := json.MarshalIndent(v, "", "\t")
if err != nil {
return nil, err
}
return Encrypt(data, passphrase)
}
// Decrypt takes a JSON-encoded ciphertext object encrypted using Encrypt and
// tries to decrypt it using passphrase. If successful, it returns the
// plaintext.
func Decrypt(ciphertext, passphrase []byte) ([]byte, error) {
data := &data{}
if err := json.Unmarshal(ciphertext, data); err != nil {
return nil, err
}
if data.KDF.Name != nameScrypt {
return nil, fmt.Errorf("encrypted: unknown kdf name %q", data.KDF.Name)
}
if data.Cipher.Name != nameSecretBox {
return nil, fmt.Errorf("encrypted: unknown cipher name %q", data.Cipher.Name)
}
if err := data.KDF.CheckParams(); err != nil {
return nil, err
}
key, err := data.KDF.Key(passphrase)
if err != nil {
return nil, err
}
return data.Cipher.Decrypt(data.Ciphertext, key)
}
// Unmarshal decrypts the data using passphrase and unmarshals the resulting
// plaintext into the value pointed to by v.
func Unmarshal(data []byte, v interface{}, passphrase []byte) error {
decrypted, err := Decrypt(data, passphrase)
if err != nil {
return err
}
return json.Unmarshal(decrypted, v)
}
func fillRandom(b []byte) error {
_, err := io.ReadFull(rand.Reader, b)
return err
}

198
vendor/golang.org/x/oauth2/deviceauth.go generated vendored Normal file
View File

@ -0,0 +1,198 @@
package oauth2
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"golang.org/x/oauth2/internal"
)
// https://datatracker.ietf.org/doc/html/rfc8628#section-3.5
const (
errAuthorizationPending = "authorization_pending"
errSlowDown = "slow_down"
errAccessDenied = "access_denied"
errExpiredToken = "expired_token"
)
// DeviceAuthResponse describes a successful RFC 8628 Device Authorization Response
// https://datatracker.ietf.org/doc/html/rfc8628#section-3.2
type DeviceAuthResponse struct {
// DeviceCode
DeviceCode string `json:"device_code"`
// UserCode is the code the user should enter at the verification uri
UserCode string `json:"user_code"`
// VerificationURI is where user should enter the user code
VerificationURI string `json:"verification_uri"`
// VerificationURIComplete (if populated) includes the user code in the verification URI. This is typically shown to the user in non-textual form, such as a QR code.
VerificationURIComplete string `json:"verification_uri_complete,omitempty"`
// Expiry is when the device code and user code expire
Expiry time.Time `json:"expires_in,omitempty"`
// Interval is the duration in seconds that Poll should wait between requests
Interval int64 `json:"interval,omitempty"`
}
func (d DeviceAuthResponse) MarshalJSON() ([]byte, error) {
type Alias DeviceAuthResponse
var expiresIn int64
if !d.Expiry.IsZero() {
expiresIn = int64(time.Until(d.Expiry).Seconds())
}
return json.Marshal(&struct {
ExpiresIn int64 `json:"expires_in,omitempty"`
*Alias
}{
ExpiresIn: expiresIn,
Alias: (*Alias)(&d),
})
}
func (c *DeviceAuthResponse) UnmarshalJSON(data []byte) error {
type Alias DeviceAuthResponse
aux := &struct {
ExpiresIn int64 `json:"expires_in"`
// workaround misspelling of verification_uri
VerificationURL string `json:"verification_url"`
*Alias
}{
Alias: (*Alias)(c),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
if aux.ExpiresIn != 0 {
c.Expiry = time.Now().UTC().Add(time.Second * time.Duration(aux.ExpiresIn))
}
if c.VerificationURI == "" {
c.VerificationURI = aux.VerificationURL
}
return nil
}
// DeviceAuth returns a device auth struct which contains a device code
// and authorization information provided for users to enter on another device.
func (c *Config) DeviceAuth(ctx context.Context, opts ...AuthCodeOption) (*DeviceAuthResponse, error) {
// https://datatracker.ietf.org/doc/html/rfc8628#section-3.1
v := url.Values{
"client_id": {c.ClientID},
}
if len(c.Scopes) > 0 {
v.Set("scope", strings.Join(c.Scopes, " "))
}
for _, opt := range opts {
opt.setValue(v)
}
return retrieveDeviceAuth(ctx, c, v)
}
func retrieveDeviceAuth(ctx context.Context, c *Config, v url.Values) (*DeviceAuthResponse, error) {
if c.Endpoint.DeviceAuthURL == "" {
return nil, errors.New("endpoint missing DeviceAuthURL")
}
req, err := http.NewRequest("POST", c.Endpoint.DeviceAuthURL, strings.NewReader(v.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
t := time.Now()
r, err := internal.ContextClient(ctx).Do(req)
if err != nil {
return nil, err
}
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
if err != nil {
return nil, fmt.Errorf("oauth2: cannot auth device: %v", err)
}
if code := r.StatusCode; code < 200 || code > 299 {
return nil, &RetrieveError{
Response: r,
Body: body,
}
}
da := &DeviceAuthResponse{}
err = json.Unmarshal(body, &da)
if err != nil {
return nil, fmt.Errorf("unmarshal %s", err)
}
if !da.Expiry.IsZero() {
// Make a small adjustment to account for time taken by the request
da.Expiry = da.Expiry.Add(-time.Since(t))
}
return da, nil
}
// DeviceAccessToken polls the server to exchange a device code for a token.
func (c *Config) DeviceAccessToken(ctx context.Context, da *DeviceAuthResponse, opts ...AuthCodeOption) (*Token, error) {
if !da.Expiry.IsZero() {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, da.Expiry)
defer cancel()
}
// https://datatracker.ietf.org/doc/html/rfc8628#section-3.4
v := url.Values{
"client_id": {c.ClientID},
"grant_type": {"urn:ietf:params:oauth:grant-type:device_code"},
"device_code": {da.DeviceCode},
}
if len(c.Scopes) > 0 {
v.Set("scope", strings.Join(c.Scopes, " "))
}
for _, opt := range opts {
opt.setValue(v)
}
// "If no value is provided, clients MUST use 5 as the default."
// https://datatracker.ietf.org/doc/html/rfc8628#section-3.2
interval := da.Interval
if interval == 0 {
interval = 5
}
ticker := time.NewTicker(time.Duration(interval) * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-ticker.C:
tok, err := retrieveToken(ctx, c, v)
if err == nil {
return tok, nil
}
e, ok := err.(*RetrieveError)
if !ok {
return nil, err
}
switch e.ErrorCode {
case errSlowDown:
// https://datatracker.ietf.org/doc/html/rfc8628#section-3.5
// "the interval MUST be increased by 5 seconds for this and all subsequent requests"
interval += 5
ticker.Reset(time.Duration(interval) * time.Second)
case errAuthorizationPending:
// Do nothing.
case errAccessDenied, errExpiredToken:
fallthrough
default:
return tok, err
}
}
}
}

29
vendor/golang.org/x/oauth2/oauth2.go generated vendored
View File

@ -75,8 +75,9 @@ type TokenSource interface {
// Endpoint represents an OAuth 2.0 provider's authorization and token
// endpoint URLs.
type Endpoint struct {
AuthURL string
TokenURL string
AuthURL string
DeviceAuthURL string
TokenURL string
// AuthStyle optionally specifies how the endpoint wants the
// client ID & client secret sent. The zero value means to
@ -143,15 +144,19 @@ func SetAuthURLParam(key, value string) AuthCodeOption {
// AuthCodeURL returns a URL to OAuth 2.0 provider's consent page
// that asks for permissions for the required scopes explicitly.
//
// State is a token to protect the user from CSRF attacks. You must
// always provide a non-empty string and validate that it matches the
// state query parameter on your redirect callback.
// See http://tools.ietf.org/html/rfc6749#section-10.12 for more info.
// State is an opaque value used by the client to maintain state between the
// request and callback. The authorization server includes this value when
// redirecting the user agent back to the client.
//
// Opts may include AccessTypeOnline or AccessTypeOffline, as well
// as ApprovalForce.
// It can also be used to pass the PKCE challenge.
// See https://www.oauth.com/oauth2-servers/pkce/ for more info.
//
// To protect against CSRF attacks, opts should include a PKCE challenge
// (S256ChallengeOption). Not all servers support PKCE. An alternative is to
// generate a random state parameter and verify it after exchange.
// See https://datatracker.ietf.org/doc/html/rfc6749#section-10.12 (predating
// PKCE), https://www.oauth.com/oauth2-servers/pkce/ and
// https://www.ietf.org/archive/id/draft-ietf-oauth-v2-1-09.html#name-cross-site-request-forgery (describing both approaches)
func (c *Config) AuthCodeURL(state string, opts ...AuthCodeOption) string {
var buf bytes.Buffer
buf.WriteString(c.Endpoint.AuthURL)
@ -166,7 +171,6 @@ func (c *Config) AuthCodeURL(state string, opts ...AuthCodeOption) string {
v.Set("scope", strings.Join(c.Scopes, " "))
}
if state != "" {
// TODO(light): Docs say never to omit state; don't allow empty.
v.Set("state", state)
}
for _, opt := range opts {
@ -211,10 +215,11 @@ func (c *Config) PasswordCredentialsToken(ctx context.Context, username, passwor
// The provided context optionally controls which HTTP client is used. See the HTTPClient variable.
//
// The code will be in the *http.Request.FormValue("code"). Before
// calling Exchange, be sure to validate FormValue("state").
// calling Exchange, be sure to validate FormValue("state") if you are
// using it to protect against CSRF attacks.
//
// Opts may include the PKCE verifier code if previously used in AuthCodeURL.
// See https://www.oauth.com/oauth2-servers/pkce/ for more info.
// If using PKCE to protect against CSRF attacks, opts should include a
// VerifierOption.
func (c *Config) Exchange(ctx context.Context, code string, opts ...AuthCodeOption) (*Token, error) {
v := url.Values{
"grant_type": {"authorization_code"},

68
vendor/golang.org/x/oauth2/pkce.go generated vendored Normal file
View File

@ -0,0 +1,68 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package oauth2
import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"net/url"
)
const (
codeChallengeKey = "code_challenge"
codeChallengeMethodKey = "code_challenge_method"
codeVerifierKey = "code_verifier"
)
// GenerateVerifier generates a PKCE code verifier with 32 octets of randomness.
// This follows recommendations in RFC 7636.
//
// A fresh verifier should be generated for each authorization.
// S256ChallengeOption(verifier) should then be passed to Config.AuthCodeURL
// (or Config.DeviceAccess) and VerifierOption(verifier) to Config.Exchange
// (or Config.DeviceAccessToken).
func GenerateVerifier() string {
// "RECOMMENDED that the output of a suitable random number generator be
// used to create a 32-octet sequence. The octet sequence is then
// base64url-encoded to produce a 43-octet URL-safe string to use as the
// code verifier."
// https://datatracker.ietf.org/doc/html/rfc7636#section-4.1
data := make([]byte, 32)
if _, err := rand.Read(data); err != nil {
panic(err)
}
return base64.RawURLEncoding.EncodeToString(data)
}
// VerifierOption returns a PKCE code verifier AuthCodeOption. It should be
// passed to Config.Exchange or Config.DeviceAccessToken only.
func VerifierOption(verifier string) AuthCodeOption {
return setParam{k: codeVerifierKey, v: verifier}
}
// S256ChallengeFromVerifier returns a PKCE code challenge derived from verifier with method S256.
//
// Prefer to use S256ChallengeOption where possible.
func S256ChallengeFromVerifier(verifier string) string {
sha := sha256.Sum256([]byte(verifier))
return base64.RawURLEncoding.EncodeToString(sha[:])
}
// S256ChallengeOption derives a PKCE code challenge derived from verifier with
// method S256. It should be passed to Config.AuthCodeURL or Config.DeviceAccess
// only.
func S256ChallengeOption(verifier string) AuthCodeOption {
return challengeOption{
challenge_method: "S256",
challenge: S256ChallengeFromVerifier(verifier),
}
}
type challengeOption struct{ challenge_method, challenge string }
func (p challengeOption) setValue(m url.Values) {
m.Set(codeChallengeMethodKey, p.challenge_method)
m.Set(codeChallengeKey, p.challenge)
}

View File

@ -188,6 +188,8 @@ type Generator struct {
trimPrefix string
lineComment bool
logf func(format string, args ...interface{}) // test logging hook; nil when not testing
}
func (g *Generator) Printf(format string, args ...interface{}) {
@ -221,13 +223,14 @@ func (g *Generator) parsePackage(patterns []string, tags []string) {
// in a separate pass? For later.
Tests: false,
BuildFlags: []string{fmt.Sprintf("-tags=%s", strings.Join(tags, " "))},
Logf: g.logf,
}
pkgs, err := packages.Load(cfg, patterns...)
if err != nil {
log.Fatal(err)
}
if len(pkgs) != 1 {
log.Fatalf("error: %d packages found", len(pkgs))
log.Fatalf("error: %d packages matching %v", len(pkgs), strings.Join(patterns, " "))
}
g.addPackage(pkgs[0])
}

View File

@ -35,7 +35,7 @@ The Package struct provides basic information about the package, including
- Imports, a map from source import strings to the Packages they name;
- Types, the type information for the package's exported symbols;
- Syntax, the parsed syntax trees for the package's source code; and
- TypeInfo, the result of a complete type-check of the package syntax trees.
- TypesInfo, the result of a complete type-check of the package syntax trees.
(See the documentation for type Package for the complete list of fields
and more detailed descriptions.)

View File

@ -9,7 +9,6 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"path"
@ -1109,7 +1108,7 @@ func (state *golistState) writeOverlays() (filename string, cleanup func(), err
if len(state.cfg.Overlay) == 0 {
return "", func() {}, nil
}
dir, err := ioutil.TempDir("", "gopackages-*")
dir, err := os.MkdirTemp("", "gopackages-*")
if err != nil {
return "", nil, err
}
@ -1128,7 +1127,7 @@ func (state *golistState) writeOverlays() (filename string, cleanup func(), err
// Create a unique filename for the overlaid files, to avoid
// creating nested directories.
noSeparator := strings.Join(strings.Split(filepath.ToSlash(k), "/"), "")
f, err := ioutil.TempFile(dir, fmt.Sprintf("*-%s", noSeparator))
f, err := os.CreateTemp(dir, fmt.Sprintf("*-%s", noSeparator))
if err != nil {
return "", func() {}, err
}
@ -1146,7 +1145,7 @@ func (state *golistState) writeOverlays() (filename string, cleanup func(), err
}
// Write out the overlay file that contains the filepath mappings.
filename = filepath.Join(dir, "overlay.json")
if err := ioutil.WriteFile(filename, b, 0665); err != nil {
if err := os.WriteFile(filename, b, 0665); err != nil {
return "", func() {}, err
}
return filename, cleanup, nil

View File

@ -16,7 +16,6 @@ import (
"go/token"
"go/types"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
@ -1127,7 +1126,7 @@ func (ld *loader) parseFile(filename string) (*ast.File, error) {
var err error
if src == nil {
ioLimit <- true // wait
src, err = ioutil.ReadFile(filename)
src, err = os.ReadFile(filename)
<-ioLimit // signal
}
if err != nil {

View File

@ -29,7 +29,6 @@ import (
"go/token"
"go/types"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
@ -221,7 +220,7 @@ func Import(packages map[string]*types.Package, path, srcDir string, lookup func
switch hdr {
case "$$B\n":
var data []byte
data, err = ioutil.ReadAll(buf)
data, err = io.ReadAll(buf)
if err != nil {
break
}

View File

@ -2,12 +2,14 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build !appengine
// +build !appengine
package internal
import (
"bytes"
"context"
"errors"
"fmt"
"io/ioutil"
@ -24,7 +26,6 @@ import (
"time"
"github.com/golang/protobuf/proto"
netcontext "golang.org/x/net/context"
basepb "google.golang.org/appengine/internal/base"
logpb "google.golang.org/appengine/internal/log"
@ -32,8 +33,7 @@ import (
)
const (
apiPath = "/rpc_http"
defaultTicketSuffix = "/default.20150612t184001.0"
apiPath = "/rpc_http"
)
var (
@ -65,21 +65,22 @@ var (
IdleConnTimeout: 90 * time.Second,
},
}
defaultTicketOnce sync.Once
defaultTicket string
backgroundContextOnce sync.Once
backgroundContext netcontext.Context
)
func apiURL() *url.URL {
func apiURL(ctx context.Context) *url.URL {
host, port := "appengine.googleapis.internal", "10001"
if h := os.Getenv("API_HOST"); h != "" {
host = h
}
if hostOverride := ctx.Value(apiHostOverrideKey); hostOverride != nil {
host = hostOverride.(string)
}
if p := os.Getenv("API_PORT"); p != "" {
port = p
}
if portOverride := ctx.Value(apiPortOverrideKey); portOverride != nil {
port = portOverride.(string)
}
return &url.URL{
Scheme: "http",
Host: host + ":" + port,
@ -87,82 +88,97 @@ func apiURL() *url.URL {
}
}
func handleHTTP(w http.ResponseWriter, r *http.Request) {
c := &context{
req: r,
outHeader: w.Header(),
apiURL: apiURL(),
}
r = r.WithContext(withContext(r.Context(), c))
c.req = r
stopFlushing := make(chan int)
// Patch up RemoteAddr so it looks reasonable.
if addr := r.Header.Get(userIPHeader); addr != "" {
r.RemoteAddr = addr
} else if addr = r.Header.Get(remoteAddrHeader); addr != "" {
r.RemoteAddr = addr
} else {
// Should not normally reach here, but pick a sensible default anyway.
r.RemoteAddr = "127.0.0.1"
}
// The address in the headers will most likely be of these forms:
// 123.123.123.123
// 2001:db8::1
// net/http.Request.RemoteAddr is specified to be in "IP:port" form.
if _, _, err := net.SplitHostPort(r.RemoteAddr); err != nil {
// Assume the remote address is only a host; add a default port.
r.RemoteAddr = net.JoinHostPort(r.RemoteAddr, "80")
}
// Start goroutine responsible for flushing app logs.
// This is done after adding c to ctx.m (and stopped before removing it)
// because flushing logs requires making an API call.
go c.logFlusher(stopFlushing)
executeRequestSafely(c, r)
c.outHeader = nil // make sure header changes aren't respected any more
stopFlushing <- 1 // any logging beyond this point will be dropped
// Flush any pending logs asynchronously.
c.pendingLogs.Lock()
flushes := c.pendingLogs.flushes
if len(c.pendingLogs.lines) > 0 {
flushes++
}
c.pendingLogs.Unlock()
flushed := make(chan struct{})
go func() {
defer close(flushed)
// Force a log flush, because with very short requests we
// may not ever flush logs.
c.flushLog(true)
}()
w.Header().Set(logFlushHeader, strconv.Itoa(flushes))
// Avoid nil Write call if c.Write is never called.
if c.outCode != 0 {
w.WriteHeader(c.outCode)
}
if c.outBody != nil {
w.Write(c.outBody)
}
// Wait for the last flush to complete before returning,
// otherwise the security ticket will not be valid.
<-flushed
// Middleware wraps an http handler so that it can make GAE API calls
func Middleware(next http.Handler) http.Handler {
return handleHTTPMiddleware(executeRequestSafelyMiddleware(next))
}
func executeRequestSafely(c *context, r *http.Request) {
defer func() {
if x := recover(); x != nil {
logf(c, 4, "%s", renderPanic(x)) // 4 == critical
c.outCode = 500
func handleHTTPMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c := &aeContext{
req: r,
outHeader: w.Header(),
}
}()
r = r.WithContext(withContext(r.Context(), c))
c.req = r
http.DefaultServeMux.ServeHTTP(c, r)
stopFlushing := make(chan int)
// Patch up RemoteAddr so it looks reasonable.
if addr := r.Header.Get(userIPHeader); addr != "" {
r.RemoteAddr = addr
} else if addr = r.Header.Get(remoteAddrHeader); addr != "" {
r.RemoteAddr = addr
} else {
// Should not normally reach here, but pick a sensible default anyway.
r.RemoteAddr = "127.0.0.1"
}
// The address in the headers will most likely be of these forms:
// 123.123.123.123
// 2001:db8::1
// net/http.Request.RemoteAddr is specified to be in "IP:port" form.
if _, _, err := net.SplitHostPort(r.RemoteAddr); err != nil {
// Assume the remote address is only a host; add a default port.
r.RemoteAddr = net.JoinHostPort(r.RemoteAddr, "80")
}
if logToLogservice() {
// Start goroutine responsible for flushing app logs.
// This is done after adding c to ctx.m (and stopped before removing it)
// because flushing logs requires making an API call.
go c.logFlusher(stopFlushing)
}
next.ServeHTTP(c, r)
c.outHeader = nil // make sure header changes aren't respected any more
flushed := make(chan struct{})
if logToLogservice() {
stopFlushing <- 1 // any logging beyond this point will be dropped
// Flush any pending logs asynchronously.
c.pendingLogs.Lock()
flushes := c.pendingLogs.flushes
if len(c.pendingLogs.lines) > 0 {
flushes++
}
c.pendingLogs.Unlock()
go func() {
defer close(flushed)
// Force a log flush, because with very short requests we
// may not ever flush logs.
c.flushLog(true)
}()
w.Header().Set(logFlushHeader, strconv.Itoa(flushes))
}
// Avoid nil Write call if c.Write is never called.
if c.outCode != 0 {
w.WriteHeader(c.outCode)
}
if c.outBody != nil {
w.Write(c.outBody)
}
if logToLogservice() {
// Wait for the last flush to complete before returning,
// otherwise the security ticket will not be valid.
<-flushed
}
})
}
func executeRequestSafelyMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if x := recover(); x != nil {
c := w.(*aeContext)
logf(c, 4, "%s", renderPanic(x)) // 4 == critical
c.outCode = 500
}
}()
next.ServeHTTP(w, r)
})
}
func renderPanic(x interface{}) string {
@ -204,9 +220,9 @@ func renderPanic(x interface{}) string {
return string(buf)
}
// context represents the context of an in-flight HTTP request.
// aeContext represents the aeContext of an in-flight HTTP request.
// It implements the appengine.Context and http.ResponseWriter interfaces.
type context struct {
type aeContext struct {
req *http.Request
outCode int
@ -218,8 +234,6 @@ type context struct {
lines []*logpb.UserAppLogLine
flushes int
}
apiURL *url.URL
}
var contextKey = "holds a *context"
@ -227,8 +241,8 @@ var contextKey = "holds a *context"
// jointContext joins two contexts in a superficial way.
// It takes values and timeouts from a base context, and only values from another context.
type jointContext struct {
base netcontext.Context
valuesOnly netcontext.Context
base context.Context
valuesOnly context.Context
}
func (c jointContext) Deadline() (time.Time, bool) {
@ -252,94 +266,54 @@ func (c jointContext) Value(key interface{}) interface{} {
// fromContext returns the App Engine context or nil if ctx is not
// derived from an App Engine context.
func fromContext(ctx netcontext.Context) *context {
c, _ := ctx.Value(&contextKey).(*context)
func fromContext(ctx context.Context) *aeContext {
c, _ := ctx.Value(&contextKey).(*aeContext)
return c
}
func withContext(parent netcontext.Context, c *context) netcontext.Context {
ctx := netcontext.WithValue(parent, &contextKey, c)
func withContext(parent context.Context, c *aeContext) context.Context {
ctx := context.WithValue(parent, &contextKey, c)
if ns := c.req.Header.Get(curNamespaceHeader); ns != "" {
ctx = withNamespace(ctx, ns)
}
return ctx
}
func toContext(c *context) netcontext.Context {
return withContext(netcontext.Background(), c)
func toContext(c *aeContext) context.Context {
return withContext(context.Background(), c)
}
func IncomingHeaders(ctx netcontext.Context) http.Header {
func IncomingHeaders(ctx context.Context) http.Header {
if c := fromContext(ctx); c != nil {
return c.req.Header
}
return nil
}
func ReqContext(req *http.Request) netcontext.Context {
func ReqContext(req *http.Request) context.Context {
return req.Context()
}
func WithContext(parent netcontext.Context, req *http.Request) netcontext.Context {
func WithContext(parent context.Context, req *http.Request) context.Context {
return jointContext{
base: parent,
valuesOnly: req.Context(),
}
}
// DefaultTicket returns a ticket used for background context or dev_appserver.
func DefaultTicket() string {
defaultTicketOnce.Do(func() {
if IsDevAppServer() {
defaultTicket = "testapp" + defaultTicketSuffix
return
}
appID := partitionlessAppID()
escAppID := strings.Replace(strings.Replace(appID, ":", "_", -1), ".", "_", -1)
majVersion := VersionID(nil)
if i := strings.Index(majVersion, "."); i > 0 {
majVersion = majVersion[:i]
}
defaultTicket = fmt.Sprintf("%s/%s.%s.%s", escAppID, ModuleName(nil), majVersion, InstanceID())
})
return defaultTicket
}
func BackgroundContext() netcontext.Context {
backgroundContextOnce.Do(func() {
// Compute background security ticket.
ticket := DefaultTicket()
c := &context{
req: &http.Request{
Header: http.Header{
ticketHeader: []string{ticket},
},
},
apiURL: apiURL(),
}
backgroundContext = toContext(c)
// TODO(dsymonds): Wire up the shutdown handler to do a final flush.
go c.logFlusher(make(chan int))
})
return backgroundContext
}
// RegisterTestRequest registers the HTTP request req for testing, such that
// any API calls are sent to the provided URL. It returns a closure to delete
// the registration.
// any API calls are sent to the provided URL.
// It should only be used by aetest package.
func RegisterTestRequest(req *http.Request, apiURL *url.URL, decorate func(netcontext.Context) netcontext.Context) (*http.Request, func()) {
c := &context{
req: req,
apiURL: apiURL,
}
ctx := withContext(decorate(req.Context()), c)
req = req.WithContext(ctx)
c.req = req
return req, func() {}
func RegisterTestRequest(req *http.Request, apiURL *url.URL, appID string) *http.Request {
ctx := req.Context()
ctx = withAPIHostOverride(ctx, apiURL.Hostname())
ctx = withAPIPortOverride(ctx, apiURL.Port())
ctx = WithAppIDOverride(ctx, appID)
// use the unregistered request as a placeholder so that withContext can read the headers
c := &aeContext{req: req}
c.req = req.WithContext(withContext(ctx, c))
return c.req
}
var errTimeout = &CallError{
@ -348,7 +322,7 @@ var errTimeout = &CallError{
Timeout: true,
}
func (c *context) Header() http.Header { return c.outHeader }
func (c *aeContext) Header() http.Header { return c.outHeader }
// Copied from $GOROOT/src/pkg/net/http/transfer.go. Some response status
// codes do not permit a response body (nor response entity headers such as
@ -365,7 +339,7 @@ func bodyAllowedForStatus(status int) bool {
return true
}
func (c *context) Write(b []byte) (int, error) {
func (c *aeContext) Write(b []byte) (int, error) {
if c.outCode == 0 {
c.WriteHeader(http.StatusOK)
}
@ -376,7 +350,7 @@ func (c *context) Write(b []byte) (int, error) {
return len(b), nil
}
func (c *context) WriteHeader(code int) {
func (c *aeContext) WriteHeader(code int) {
if c.outCode != 0 {
logf(c, 3, "WriteHeader called multiple times on request.") // error level
return
@ -384,10 +358,11 @@ func (c *context) WriteHeader(code int) {
c.outCode = code
}
func (c *context) post(body []byte, timeout time.Duration) (b []byte, err error) {
func post(ctx context.Context, body []byte, timeout time.Duration) (b []byte, err error) {
apiURL := apiURL(ctx)
hreq := &http.Request{
Method: "POST",
URL: c.apiURL,
URL: apiURL,
Header: http.Header{
apiEndpointHeader: apiEndpointHeaderValue,
apiMethodHeader: apiMethodHeaderValue,
@ -396,13 +371,16 @@ func (c *context) post(body []byte, timeout time.Duration) (b []byte, err error)
},
Body: ioutil.NopCloser(bytes.NewReader(body)),
ContentLength: int64(len(body)),
Host: c.apiURL.Host,
Host: apiURL.Host,
}
if info := c.req.Header.Get(dapperHeader); info != "" {
hreq.Header.Set(dapperHeader, info)
}
if info := c.req.Header.Get(traceHeader); info != "" {
hreq.Header.Set(traceHeader, info)
c := fromContext(ctx)
if c != nil {
if info := c.req.Header.Get(dapperHeader); info != "" {
hreq.Header.Set(dapperHeader, info)
}
if info := c.req.Header.Get(traceHeader); info != "" {
hreq.Header.Set(traceHeader, info)
}
}
tr := apiHTTPClient.Transport.(*http.Transport)
@ -444,7 +422,7 @@ func (c *context) post(body []byte, timeout time.Duration) (b []byte, err error)
return hrespBody, nil
}
func Call(ctx netcontext.Context, service, method string, in, out proto.Message) error {
func Call(ctx context.Context, service, method string, in, out proto.Message) error {
if ns := NamespaceFromContext(ctx); ns != "" {
if fn, ok := NamespaceMods[service]; ok {
fn(in, ns)
@ -463,15 +441,11 @@ func Call(ctx netcontext.Context, service, method string, in, out proto.Message)
}
c := fromContext(ctx)
if c == nil {
// Give a good error message rather than a panic lower down.
return errNotAppEngineContext
}
// Apply transaction modifications if we're in a transaction.
if t := transactionFromContext(ctx); t != nil {
if t.finished {
return errors.New("transaction context has expired")
return errors.New("transaction aeContext has expired")
}
applyTransaction(in, &t.transaction)
}
@ -487,20 +461,13 @@ func Call(ctx netcontext.Context, service, method string, in, out proto.Message)
return err
}
ticket := c.req.Header.Get(ticketHeader)
// Use a test ticket under test environment.
if ticket == "" {
if appid := ctx.Value(&appIDOverrideKey); appid != nil {
ticket = appid.(string) + defaultTicketSuffix
ticket := ""
if c != nil {
ticket = c.req.Header.Get(ticketHeader)
if dri := c.req.Header.Get(devRequestIdHeader); IsDevAppServer() && dri != "" {
ticket = dri
}
}
// Fall back to use background ticket when the request ticket is not available in Flex or dev_appserver.
if ticket == "" {
ticket = DefaultTicket()
}
if dri := c.req.Header.Get(devRequestIdHeader); IsDevAppServer() && dri != "" {
ticket = dri
}
req := &remotepb.Request{
ServiceName: &service,
Method: &method,
@ -512,7 +479,7 @@ func Call(ctx netcontext.Context, service, method string, in, out proto.Message)
return err
}
hrespBody, err := c.post(hreqBody, timeout)
hrespBody, err := post(ctx, hreqBody, timeout)
if err != nil {
return err
}
@ -549,11 +516,11 @@ func Call(ctx netcontext.Context, service, method string, in, out proto.Message)
return proto.Unmarshal(res.Response, out)
}
func (c *context) Request() *http.Request {
func (c *aeContext) Request() *http.Request {
return c.req
}
func (c *context) addLogLine(ll *logpb.UserAppLogLine) {
func (c *aeContext) addLogLine(ll *logpb.UserAppLogLine) {
// Truncate long log lines.
// TODO(dsymonds): Check if this is still necessary.
const lim = 8 << 10
@ -575,18 +542,20 @@ var logLevelName = map[int64]string{
4: "CRITICAL",
}
func logf(c *context, level int64, format string, args ...interface{}) {
func logf(c *aeContext, level int64, format string, args ...interface{}) {
if c == nil {
panic("not an App Engine context")
panic("not an App Engine aeContext")
}
s := fmt.Sprintf(format, args...)
s = strings.TrimRight(s, "\n") // Remove any trailing newline characters.
c.addLogLine(&logpb.UserAppLogLine{
TimestampUsec: proto.Int64(time.Now().UnixNano() / 1e3),
Level: &level,
Message: &s,
})
// Only duplicate log to stderr if not running on App Engine second generation
if logToLogservice() {
c.addLogLine(&logpb.UserAppLogLine{
TimestampUsec: proto.Int64(time.Now().UnixNano() / 1e3),
Level: &level,
Message: &s,
})
}
// Log to stdout if not deployed
if !IsSecondGen() {
log.Print(logLevelName[level] + ": " + s)
}
@ -594,7 +563,7 @@ func logf(c *context, level int64, format string, args ...interface{}) {
// flushLog attempts to flush any pending logs to the appserver.
// It should not be called concurrently.
func (c *context) flushLog(force bool) (flushed bool) {
func (c *aeContext) flushLog(force bool) (flushed bool) {
c.pendingLogs.Lock()
// Grab up to 30 MB. We can get away with up to 32 MB, but let's be cautious.
n, rem := 0, 30<<20
@ -655,7 +624,7 @@ const (
forceFlushInterval = 60 * time.Second
)
func (c *context) logFlusher(stop <-chan int) {
func (c *aeContext) logFlusher(stop <-chan int) {
lastFlush := time.Now()
tick := time.NewTicker(flushInterval)
for {
@ -673,6 +642,12 @@ func (c *context) logFlusher(stop <-chan int) {
}
}
func ContextForTesting(req *http.Request) netcontext.Context {
return toContext(&context{req: req})
func ContextForTesting(req *http.Request) context.Context {
return toContext(&aeContext{req: req})
}
func logToLogservice() bool {
// TODO: replace logservice with json structured logs to $LOG_DIR/app.log.json
// where $LOG_DIR is /var/log in prod and some tmpdir in dev
return os.Getenv("LOG_TO_LOGSERVICE") != "0"
}

View File

@ -2,11 +2,13 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build appengine
// +build appengine
package internal
import (
"context"
"errors"
"fmt"
"net/http"
@ -17,20 +19,19 @@ import (
basepb "appengine_internal/base"
"github.com/golang/protobuf/proto"
netcontext "golang.org/x/net/context"
)
var contextKey = "holds an appengine.Context"
// fromContext returns the App Engine context or nil if ctx is not
// derived from an App Engine context.
func fromContext(ctx netcontext.Context) appengine.Context {
func fromContext(ctx context.Context) appengine.Context {
c, _ := ctx.Value(&contextKey).(appengine.Context)
return c
}
// This is only for classic App Engine adapters.
func ClassicContextFromContext(ctx netcontext.Context) (appengine.Context, error) {
func ClassicContextFromContext(ctx context.Context) (appengine.Context, error) {
c := fromContext(ctx)
if c == nil {
return nil, errNotAppEngineContext
@ -38,8 +39,8 @@ func ClassicContextFromContext(ctx netcontext.Context) (appengine.Context, error
return c, nil
}
func withContext(parent netcontext.Context, c appengine.Context) netcontext.Context {
ctx := netcontext.WithValue(parent, &contextKey, c)
func withContext(parent context.Context, c appengine.Context) context.Context {
ctx := context.WithValue(parent, &contextKey, c)
s := &basepb.StringProto{}
c.Call("__go__", "GetNamespace", &basepb.VoidProto{}, s, nil)
@ -50,7 +51,7 @@ func withContext(parent netcontext.Context, c appengine.Context) netcontext.Cont
return ctx
}
func IncomingHeaders(ctx netcontext.Context) http.Header {
func IncomingHeaders(ctx context.Context) http.Header {
if c := fromContext(ctx); c != nil {
if req, ok := c.Request().(*http.Request); ok {
return req.Header
@ -59,11 +60,11 @@ func IncomingHeaders(ctx netcontext.Context) http.Header {
return nil
}
func ReqContext(req *http.Request) netcontext.Context {
return WithContext(netcontext.Background(), req)
func ReqContext(req *http.Request) context.Context {
return WithContext(context.Background(), req)
}
func WithContext(parent netcontext.Context, req *http.Request) netcontext.Context {
func WithContext(parent context.Context, req *http.Request) context.Context {
c := appengine.NewContext(req)
return withContext(parent, c)
}
@ -83,11 +84,11 @@ func (t *testingContext) Call(service, method string, _, _ appengine_internal.Pr
}
func (t *testingContext) Request() interface{} { return t.req }
func ContextForTesting(req *http.Request) netcontext.Context {
return withContext(netcontext.Background(), &testingContext{req: req})
func ContextForTesting(req *http.Request) context.Context {
return withContext(context.Background(), &testingContext{req: req})
}
func Call(ctx netcontext.Context, service, method string, in, out proto.Message) error {
func Call(ctx context.Context, service, method string, in, out proto.Message) error {
if ns := NamespaceFromContext(ctx); ns != "" {
if fn, ok := NamespaceMods[service]; ok {
fn(in, ns)
@ -144,8 +145,8 @@ func Call(ctx netcontext.Context, service, method string, in, out proto.Message)
return err
}
func handleHTTP(w http.ResponseWriter, r *http.Request) {
panic("handleHTTP called; this should be impossible")
func Middleware(next http.Handler) http.Handler {
panic("Middleware called; this should be impossible")
}
func logf(c appengine.Context, level int64, format string, args ...interface{}) {

View File

@ -5,20 +5,26 @@
package internal
import (
"context"
"errors"
"os"
"github.com/golang/protobuf/proto"
netcontext "golang.org/x/net/context"
)
type ctxKey string
func (c ctxKey) String() string {
return "appengine context key: " + string(c)
}
var errNotAppEngineContext = errors.New("not an App Engine context")
type CallOverrideFunc func(ctx netcontext.Context, service, method string, in, out proto.Message) error
type CallOverrideFunc func(ctx context.Context, service, method string, in, out proto.Message) error
var callOverrideKey = "holds []CallOverrideFunc"
func WithCallOverride(ctx netcontext.Context, f CallOverrideFunc) netcontext.Context {
func WithCallOverride(ctx context.Context, f CallOverrideFunc) context.Context {
// We avoid appending to any existing call override
// so we don't risk overwriting a popped stack below.
var cofs []CallOverrideFunc
@ -26,10 +32,10 @@ func WithCallOverride(ctx netcontext.Context, f CallOverrideFunc) netcontext.Con
cofs = append(cofs, uf...)
}
cofs = append(cofs, f)
return netcontext.WithValue(ctx, &callOverrideKey, cofs)
return context.WithValue(ctx, &callOverrideKey, cofs)
}
func callOverrideFromContext(ctx netcontext.Context) (CallOverrideFunc, netcontext.Context, bool) {
func callOverrideFromContext(ctx context.Context) (CallOverrideFunc, context.Context, bool) {
cofs, _ := ctx.Value(&callOverrideKey).([]CallOverrideFunc)
if len(cofs) == 0 {
return nil, nil, false
@ -37,7 +43,7 @@ func callOverrideFromContext(ctx netcontext.Context) (CallOverrideFunc, netconte
// We found a list of overrides; grab the last, and reconstitute a
// context that will hide it.
f := cofs[len(cofs)-1]
ctx = netcontext.WithValue(ctx, &callOverrideKey, cofs[:len(cofs)-1])
ctx = context.WithValue(ctx, &callOverrideKey, cofs[:len(cofs)-1])
return f, ctx, true
}
@ -45,23 +51,35 @@ type logOverrideFunc func(level int64, format string, args ...interface{})
var logOverrideKey = "holds a logOverrideFunc"
func WithLogOverride(ctx netcontext.Context, f logOverrideFunc) netcontext.Context {
return netcontext.WithValue(ctx, &logOverrideKey, f)
func WithLogOverride(ctx context.Context, f logOverrideFunc) context.Context {
return context.WithValue(ctx, &logOverrideKey, f)
}
var appIDOverrideKey = "holds a string, being the full app ID"
func WithAppIDOverride(ctx netcontext.Context, appID string) netcontext.Context {
return netcontext.WithValue(ctx, &appIDOverrideKey, appID)
func WithAppIDOverride(ctx context.Context, appID string) context.Context {
return context.WithValue(ctx, &appIDOverrideKey, appID)
}
var apiHostOverrideKey = ctxKey("holds a string, being the alternate API_HOST")
func withAPIHostOverride(ctx context.Context, apiHost string) context.Context {
return context.WithValue(ctx, apiHostOverrideKey, apiHost)
}
var apiPortOverrideKey = ctxKey("holds a string, being the alternate API_PORT")
func withAPIPortOverride(ctx context.Context, apiPort string) context.Context {
return context.WithValue(ctx, apiPortOverrideKey, apiPort)
}
var namespaceKey = "holds the namespace string"
func withNamespace(ctx netcontext.Context, ns string) netcontext.Context {
return netcontext.WithValue(ctx, &namespaceKey, ns)
func withNamespace(ctx context.Context, ns string) context.Context {
return context.WithValue(ctx, &namespaceKey, ns)
}
func NamespaceFromContext(ctx netcontext.Context) string {
func NamespaceFromContext(ctx context.Context) string {
// If there's no namespace, return the empty string.
ns, _ := ctx.Value(&namespaceKey).(string)
return ns
@ -70,14 +88,14 @@ func NamespaceFromContext(ctx netcontext.Context) string {
// FullyQualifiedAppID returns the fully-qualified application ID.
// This may contain a partition prefix (e.g. "s~" for High Replication apps),
// or a domain prefix (e.g. "example.com:").
func FullyQualifiedAppID(ctx netcontext.Context) string {
func FullyQualifiedAppID(ctx context.Context) string {
if id, ok := ctx.Value(&appIDOverrideKey).(string); ok {
return id
}
return fullyQualifiedAppID(ctx)
}
func Logf(ctx netcontext.Context, level int64, format string, args ...interface{}) {
func Logf(ctx context.Context, level int64, format string, args ...interface{}) {
if f, ok := ctx.Value(&logOverrideKey).(logOverrideFunc); ok {
f(level, format, args...)
return
@ -90,7 +108,7 @@ func Logf(ctx netcontext.Context, level int64, format string, args ...interface{
}
// NamespacedContext wraps a Context to support namespaces.
func NamespacedContext(ctx netcontext.Context, namespace string) netcontext.Context {
func NamespacedContext(ctx context.Context, namespace string) context.Context {
return withNamespace(ctx, namespace)
}

View File

@ -5,9 +5,8 @@
package internal
import (
"context"
"os"
netcontext "golang.org/x/net/context"
)
var (
@ -23,7 +22,7 @@ var (
// AppID is the implementation of the wrapper function of the same name in
// ../identity.go. See that file for commentary.
func AppID(c netcontext.Context) string {
func AppID(c context.Context) string {
return appID(FullyQualifiedAppID(c))
}
@ -35,7 +34,7 @@ func IsStandard() bool {
return appengineStandard || IsSecondGen()
}
// IsStandard is the implementation of the wrapper function of the same name in
// IsSecondGen is the implementation of the wrapper function of the same name in
// ../appengine.go. See that file for commentary.
func IsSecondGen() bool {
// Second-gen runtimes set $GAE_ENV so we use that to check if we're on a second-gen runtime.

View File

@ -2,21 +2,22 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build appengine
// +build appengine
package internal
import (
"appengine"
"context"
netcontext "golang.org/x/net/context"
"appengine"
)
func init() {
appengineStandard = true
}
func DefaultVersionHostname(ctx netcontext.Context) string {
func DefaultVersionHostname(ctx context.Context) string {
c := fromContext(ctx)
if c == nil {
panic(errNotAppEngineContext)
@ -24,12 +25,12 @@ func DefaultVersionHostname(ctx netcontext.Context) string {
return appengine.DefaultVersionHostname(c)
}
func Datacenter(_ netcontext.Context) string { return appengine.Datacenter() }
func ServerSoftware() string { return appengine.ServerSoftware() }
func InstanceID() string { return appengine.InstanceID() }
func IsDevAppServer() bool { return appengine.IsDevAppServer() }
func Datacenter(_ context.Context) string { return appengine.Datacenter() }
func ServerSoftware() string { return appengine.ServerSoftware() }
func InstanceID() string { return appengine.InstanceID() }
func IsDevAppServer() bool { return appengine.IsDevAppServer() }
func RequestID(ctx netcontext.Context) string {
func RequestID(ctx context.Context) string {
c := fromContext(ctx)
if c == nil {
panic(errNotAppEngineContext)
@ -37,14 +38,14 @@ func RequestID(ctx netcontext.Context) string {
return appengine.RequestID(c)
}
func ModuleName(ctx netcontext.Context) string {
func ModuleName(ctx context.Context) string {
c := fromContext(ctx)
if c == nil {
panic(errNotAppEngineContext)
}
return appengine.ModuleName(c)
}
func VersionID(ctx netcontext.Context) string {
func VersionID(ctx context.Context) string {
c := fromContext(ctx)
if c == nil {
panic(errNotAppEngineContext)
@ -52,7 +53,7 @@ func VersionID(ctx netcontext.Context) string {
return appengine.VersionID(c)
}
func fullyQualifiedAppID(ctx netcontext.Context) string {
func fullyQualifiedAppID(ctx context.Context) string {
c := fromContext(ctx)
if c == nil {
panic(errNotAppEngineContext)

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build appenginevm
// +build appenginevm
package internal

View File

@ -2,17 +2,17 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build !appengine
// +build !appengine
package internal
import (
"context"
"log"
"net/http"
"os"
"strings"
netcontext "golang.org/x/net/context"
)
// These functions are implementations of the wrapper functions
@ -24,7 +24,7 @@ const (
hDatacenter = "X-AppEngine-Datacenter"
)
func ctxHeaders(ctx netcontext.Context) http.Header {
func ctxHeaders(ctx context.Context) http.Header {
c := fromContext(ctx)
if c == nil {
return nil
@ -32,15 +32,15 @@ func ctxHeaders(ctx netcontext.Context) http.Header {
return c.Request().Header
}
func DefaultVersionHostname(ctx netcontext.Context) string {
func DefaultVersionHostname(ctx context.Context) string {
return ctxHeaders(ctx).Get(hDefaultVersionHostname)
}
func RequestID(ctx netcontext.Context) string {
func RequestID(ctx context.Context) string {
return ctxHeaders(ctx).Get(hRequestLogId)
}
func Datacenter(ctx netcontext.Context) string {
func Datacenter(ctx context.Context) string {
if dc := ctxHeaders(ctx).Get(hDatacenter); dc != "" {
return dc
}
@ -71,7 +71,7 @@ func ServerSoftware() string {
// TODO(dsymonds): Remove the metadata fetches.
func ModuleName(_ netcontext.Context) string {
func ModuleName(_ context.Context) string {
if s := os.Getenv("GAE_MODULE_NAME"); s != "" {
return s
}
@ -81,7 +81,7 @@ func ModuleName(_ netcontext.Context) string {
return string(mustGetMetadata("instance/attributes/gae_backend_name"))
}
func VersionID(_ netcontext.Context) string {
func VersionID(_ context.Context) string {
if s1, s2 := os.Getenv("GAE_MODULE_VERSION"), os.Getenv("GAE_MINOR_VERSION"); s1 != "" && s2 != "" {
return s1 + "." + s2
}
@ -112,7 +112,7 @@ func partitionlessAppID() string {
return string(mustGetMetadata("instance/attributes/gae_project"))
}
func fullyQualifiedAppID(_ netcontext.Context) string {
func fullyQualifiedAppID(_ context.Context) string {
if s := os.Getenv("GAE_APPLICATION"); s != "" {
return s
}
@ -130,5 +130,5 @@ func fullyQualifiedAppID(_ netcontext.Context) string {
}
func IsDevAppServer() bool {
return os.Getenv("RUN_WITH_DEVAPPSERVER") != ""
return os.Getenv("RUN_WITH_DEVAPPSERVER") != "" || os.Getenv("GAE_ENV") == "localdev"
}

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build appengine
// +build appengine
package internal

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
//go:build !appengine
// +build !appengine
package internal
@ -29,7 +30,7 @@ func Main() {
if IsDevAppServer() {
host = "127.0.0.1"
}
if err := http.ListenAndServe(host+":"+port, http.HandlerFunc(handleHTTP)); err != nil {
if err := http.ListenAndServe(host+":"+port, Middleware(http.DefaultServeMux)); err != nil {
log.Fatalf("http.ListenAndServe: %v", err)
}
}

View File

@ -7,11 +7,11 @@ package internal
// This file implements hooks for applying datastore transactions.
import (
"context"
"errors"
"reflect"
"github.com/golang/protobuf/proto"
netcontext "golang.org/x/net/context"
basepb "google.golang.org/appengine/internal/base"
pb "google.golang.org/appengine/internal/datastore"
@ -38,13 +38,13 @@ func applyTransaction(pb proto.Message, t *pb.Transaction) {
var transactionKey = "used for *Transaction"
func transactionFromContext(ctx netcontext.Context) *transaction {
func transactionFromContext(ctx context.Context) *transaction {
t, _ := ctx.Value(&transactionKey).(*transaction)
return t
}
func withTransaction(ctx netcontext.Context, t *transaction) netcontext.Context {
return netcontext.WithValue(ctx, &transactionKey, t)
func withTransaction(ctx context.Context, t *transaction) context.Context {
return context.WithValue(ctx, &transactionKey, t)
}
type transaction struct {
@ -54,7 +54,7 @@ type transaction struct {
var ErrConcurrentTransaction = errors.New("internal: concurrent transaction")
func RunTransactionOnce(c netcontext.Context, f func(netcontext.Context) error, xg bool, readOnly bool, previousTransaction *pb.Transaction) (*pb.Transaction, error) {
func RunTransactionOnce(c context.Context, f func(context.Context) error, xg bool, readOnly bool, previousTransaction *pb.Transaction) (*pb.Transaction, error) {
if transactionFromContext(c) != nil {
return nil, errors.New("nested transactions are not supported")
}

View File

@ -7,6 +7,7 @@
package urlfetch // import "google.golang.org/appengine/urlfetch"
import (
"context"
"errors"
"fmt"
"io"
@ -18,7 +19,6 @@ import (
"time"
"github.com/golang/protobuf/proto"
"golang.org/x/net/context"
"google.golang.org/appengine/internal"
pb "google.golang.org/appengine/internal/urlfetch"
@ -44,11 +44,10 @@ type Transport struct {
var _ http.RoundTripper = (*Transport)(nil)
// Client returns an *http.Client using a default urlfetch Transport. This
// client will have the default deadline of 5 seconds, and will check the
// validity of SSL certificates.
// client will check the validity of SSL certificates.
//
// Any deadline of the provided context will be used for requests through this client;
// if the client does not have a deadline then a 5 second default is used.
// Any deadline of the provided context will be used for requests through this client.
// If the client does not have a deadline, then an App Engine default of 60 second is used.
func Client(ctx context.Context) *http.Client {
return &http.Client{
Transport: &Transport{

View File

@ -14,21 +14,14 @@ RPC framework that puts mobile and HTTP/2 first. For more information see the
## Installation
With [Go module][] support (Go 1.11+), simply add the following import
Simply add the following import to your code, and then `go [build|run|test]`
will automatically fetch the necessary dependencies:
```go
import "google.golang.org/grpc"
```
to your code, and then `go [build|run|test]` will automatically fetch the
necessary dependencies.
Otherwise, to install the `grpc-go` package, run the following command:
```console
$ go get -u google.golang.org/grpc
```
> **Note:** If you are trying to access `grpc-go` from **China**, see the
> [FAQ](#FAQ) below.
@ -56,15 +49,6 @@ To build Go code, there are several options:
- Set up a VPN and access google.golang.org through that.
- Without Go module support: `git clone` the repo manually:
```sh
git clone https://github.com/grpc/grpc-go.git $GOPATH/src/google.golang.org/grpc
```
You will need to do the same for all of grpc's dependencies in `golang.org`,
e.g. `golang.org/x/net`.
- With Go module support: it is possible to use the `replace` feature of `go
mod` to create aliases for golang.org packages. In your project's directory:
@ -76,33 +60,13 @@ To build Go code, there are several options:
```
Again, this will need to be done for all transitive dependencies hosted on
golang.org as well. For details, refer to [golang/go issue #28652](https://github.com/golang/go/issues/28652).
golang.org as well. For details, refer to [golang/go issue
#28652](https://github.com/golang/go/issues/28652).
### Compiling error, undefined: grpc.SupportPackageIsVersion
#### If you are using Go modules:
Ensure your gRPC-Go version is `require`d at the appropriate version in
the same module containing the generated `.pb.go` files. For example,
`SupportPackageIsVersion6` needs `v1.27.0`, so in your `go.mod` file:
```go
module <your module name>
require (
google.golang.org/grpc v1.27.0
)
```
#### If you are *not* using Go modules:
Update the `proto` package, gRPC package, and rebuild the `.proto` files:
```sh
go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
go get -u google.golang.org/grpc
protoc --go_out=plugins=grpc:. *.proto
```
Please update to the latest version of gRPC-Go using
`go get google.golang.org/grpc`.
### How to turn on logging
@ -121,9 +85,11 @@ possible reasons, including:
1. mis-configured transport credentials, connection failed on handshaking
1. bytes disrupted, possibly by a proxy in between
1. server shutdown
1. Keepalive parameters caused connection shutdown, for example if you have configured
your server to terminate connections regularly to [trigger DNS lookups](https://github.com/grpc/grpc-go/issues/3170#issuecomment-552517779).
If this is the case, you may want to increase your [MaxConnectionAgeGrace](https://pkg.go.dev/google.golang.org/grpc/keepalive?tab=doc#ServerParameters),
1. Keepalive parameters caused connection shutdown, for example if you have
configured your server to terminate connections regularly to [trigger DNS
lookups](https://github.com/grpc/grpc-go/issues/3170#issuecomment-552517779).
If this is the case, you may want to increase your
[MaxConnectionAgeGrace](https://pkg.go.dev/google.golang.org/grpc/keepalive?tab=doc#ServerParameters),
to allow longer RPC calls to finish.
It can be tricky to debug this because the error happens on the client side but

View File

@ -34,26 +34,26 @@ import (
// key/value pairs. Keys must be hashable, and users should define their own
// types for keys. Values should not be modified after they are added to an
// Attributes or if they were received from one. If values implement 'Equal(o
// interface{}) bool', it will be called by (*Attributes).Equal to determine
// whether two values with the same key should be considered equal.
// any) bool', it will be called by (*Attributes).Equal to determine whether
// two values with the same key should be considered equal.
type Attributes struct {
m map[interface{}]interface{}
m map[any]any
}
// New returns a new Attributes containing the key/value pair.
func New(key, value interface{}) *Attributes {
return &Attributes{m: map[interface{}]interface{}{key: value}}
func New(key, value any) *Attributes {
return &Attributes{m: map[any]any{key: value}}
}
// WithValue returns a new Attributes containing the previous keys and values
// and the new key/value pair. If the same key appears multiple times, the
// last value overwrites all previous values for that key. To remove an
// existing key, use a nil value. value should not be modified later.
func (a *Attributes) WithValue(key, value interface{}) *Attributes {
func (a *Attributes) WithValue(key, value any) *Attributes {
if a == nil {
return New(key, value)
}
n := &Attributes{m: make(map[interface{}]interface{}, len(a.m)+1)}
n := &Attributes{m: make(map[any]any, len(a.m)+1)}
for k, v := range a.m {
n.m[k] = v
}
@ -63,20 +63,19 @@ func (a *Attributes) WithValue(key, value interface{}) *Attributes {
// Value returns the value associated with these attributes for key, or nil if
// no value is associated with key. The returned value should not be modified.
func (a *Attributes) Value(key interface{}) interface{} {
func (a *Attributes) Value(key any) any {
if a == nil {
return nil
}
return a.m[key]
}
// Equal returns whether a and o are equivalent. If 'Equal(o interface{})
// bool' is implemented for a value in the attributes, it is called to
// determine if the value matches the one stored in the other attributes. If
// Equal is not implemented, standard equality is used to determine if the two
// values are equal. Note that some types (e.g. maps) aren't comparable by
// default, so they must be wrapped in a struct, or in an alias type, with Equal
// defined.
// Equal returns whether a and o are equivalent. If 'Equal(o any) bool' is
// implemented for a value in the attributes, it is called to determine if the
// value matches the one stored in the other attributes. If Equal is not
// implemented, standard equality is used to determine if the two values are
// equal. Note that some types (e.g. maps) aren't comparable by default, so
// they must be wrapped in a struct, or in an alias type, with Equal defined.
func (a *Attributes) Equal(o *Attributes) bool {
if a == nil && o == nil {
return true
@ -93,7 +92,7 @@ func (a *Attributes) Equal(o *Attributes) bool {
// o missing element of a
return false
}
if eq, ok := v.(interface{ Equal(o interface{}) bool }); ok {
if eq, ok := v.(interface{ Equal(o any) bool }); ok {
if !eq.Equal(ov) {
return false
}
@ -122,7 +121,7 @@ func (a *Attributes) String() string {
return sb.String()
}
func str(x interface{}) string {
func str(x any) string {
if v, ok := x.(fmt.Stringer); ok {
return v.String()
} else if v, ok := x.(string); ok {

View File

@ -105,8 +105,8 @@ type SubConn interface {
//
// This will trigger a state transition for the SubConn.
//
// Deprecated: This method is now part of the ClientConn interface and will
// eventually be removed from here.
// Deprecated: this method will be removed. Create new SubConns for new
// addresses instead.
UpdateAddresses([]resolver.Address)
// Connect starts the connecting for this SubConn.
Connect()
@ -115,6 +115,13 @@ type SubConn interface {
// creates a new one and returns it. Returns a close function which must
// be called when the Producer is no longer needed.
GetOrBuildProducer(ProducerBuilder) (p Producer, close func())
// Shutdown shuts down the SubConn gracefully. Any started RPCs will be
// allowed to complete. No future calls should be made on the SubConn.
// One final state update will be delivered to the StateListener (or
// UpdateSubConnState; deprecated) with ConnectivityState of Shutdown to
// indicate the shutdown operation. This may be delivered before
// in-progress RPCs are complete and the actual connection is closed.
Shutdown()
}
// NewSubConnOptions contains options to create new SubConn.
@ -129,6 +136,11 @@ type NewSubConnOptions struct {
// HealthCheckEnabled indicates whether health check service should be
// enabled on this SubConn
HealthCheckEnabled bool
// StateListener is called when the state of the subconn changes. If nil,
// Balancer.UpdateSubConnState will be called instead. Will never be
// invoked until after Connect() is called on the SubConn created with
// these options.
StateListener func(SubConnState)
}
// State contains the balancer's state relevant to the gRPC ClientConn.
@ -150,16 +162,24 @@ type ClientConn interface {
// NewSubConn is called by balancer to create a new SubConn.
// It doesn't block and wait for the connections to be established.
// Behaviors of the SubConn can be controlled by options.
//
// Deprecated: please be aware that in a future version, SubConns will only
// support one address per SubConn.
NewSubConn([]resolver.Address, NewSubConnOptions) (SubConn, error)
// RemoveSubConn removes the SubConn from ClientConn.
// The SubConn will be shutdown.
//
// Deprecated: use SubConn.Shutdown instead.
RemoveSubConn(SubConn)
// UpdateAddresses updates the addresses used in the passed in SubConn.
// gRPC checks if the currently connected address is still in the new list.
// If so, the connection will be kept. Else, the connection will be
// gracefully closed, and a new connection will be created.
//
// This will trigger a state transition for the SubConn.
// This may trigger a state transition for the SubConn.
//
// Deprecated: this method will be removed. Create new SubConns for new
// addresses instead.
UpdateAddresses(SubConn, []resolver.Address)
// UpdateState notifies gRPC that the balancer's internal state has
@ -250,7 +270,7 @@ type DoneInfo struct {
// trailing metadata.
//
// The only supported type now is *orca_v3.LoadReport.
ServerLoad interface{}
ServerLoad any
}
var (
@ -343,9 +363,13 @@ type Balancer interface {
ResolverError(error)
// UpdateSubConnState is called by gRPC when the state of a SubConn
// changes.
//
// Deprecated: Use NewSubConnOptions.StateListener when creating the
// SubConn instead.
UpdateSubConnState(SubConn, SubConnState)
// Close closes the balancer. The balancer is not required to call
// ClientConn.RemoveSubConn for its existing SubConns.
// Close closes the balancer. The balancer is not currently required to
// call SubConn.Shutdown for its existing SubConns; however, this will be
// required in a future release, so it is recommended.
Close()
}
@ -390,15 +414,14 @@ var ErrBadResolverState = errors.New("bad resolver state")
type ProducerBuilder interface {
// Build creates a Producer. The first parameter is always a
// grpc.ClientConnInterface (a type to allow creating RPCs/streams on the
// associated SubConn), but is declared as interface{} to avoid a
// dependency cycle. Should also return a close function that will be
// called when all references to the Producer have been given up.
Build(grpcClientConnInterface interface{}) (p Producer, close func())
// associated SubConn), but is declared as `any` to avoid a dependency
// cycle. Should also return a close function that will be called when all
// references to the Producer have been given up.
Build(grpcClientConnInterface any) (p Producer, close func())
}
// A Producer is a type shared among potentially many consumers. It is
// associated with a SubConn, and an implementation will typically contain
// other methods to provide additional functionality, e.g. configuration or
// subscription registration.
type Producer interface {
}
type Producer any

View File

@ -105,7 +105,12 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
addrsSet.Set(a, nil)
if _, ok := b.subConns.Get(a); !ok {
// a is a new address (not existing in b.subConns).
sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: b.config.HealthCheck})
var sc balancer.SubConn
opts := balancer.NewSubConnOptions{
HealthCheckEnabled: b.config.HealthCheck,
StateListener: func(scs balancer.SubConnState) { b.updateSubConnState(sc, scs) },
}
sc, err := b.cc.NewSubConn([]resolver.Address{a}, opts)
if err != nil {
logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err)
continue
@ -121,10 +126,10 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
sc := sci.(balancer.SubConn)
// a was removed by resolver.
if _, ok := addrsSet.Get(a); !ok {
b.cc.RemoveSubConn(sc)
sc.Shutdown()
b.subConns.Delete(a)
// Keep the state of this sc in b.scStates until sc's state becomes Shutdown.
// The entry will be deleted in UpdateSubConnState.
// The entry will be deleted in updateSubConnState.
}
}
// If resolver state contains no addresses, return an error so ClientConn
@ -177,7 +182,12 @@ func (b *baseBalancer) regeneratePicker() {
b.picker = b.pickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs})
}
// UpdateSubConnState is a nop because a StateListener is always set in NewSubConn.
func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
logger.Errorf("base.baseBalancer: UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
}
func (b *baseBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
s := state.ConnectivityState
if logger.V(2) {
logger.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s)
@ -204,8 +214,8 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su
case connectivity.Idle:
sc.Connect()
case connectivity.Shutdown:
// When an address was removed by resolver, b called RemoveSubConn but
// kept the sc's state in scStates. Remove state for this sc here.
// When an address was removed by resolver, b called Shutdown but kept
// the sc's state in scStates. Remove state for this sc here.
delete(b.scStates, sc)
case connectivity.TransientFailure:
// Save error to be reported via picker.
@ -226,7 +236,7 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su
}
// Close is a nop because base balancer doesn't have internal state to clean up,
// and it doesn't need to call RemoveSubConn for the SubConns.
// and it doesn't need to call Shutdown for the SubConns.
func (b *baseBalancer) Close() {
}

View File

@ -99,20 +99,6 @@ func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnStat
// lock held. But the lock guards only the scheduling part. The actual
// callback is called asynchronously without the lock being held.
ok := ccb.serializer.Schedule(func(_ context.Context) {
// If the addresses specified in the update contain addresses of type
// "grpclb" and the selected LB policy is not "grpclb", these addresses
// will be filtered out and ccs will be modified with the updated
// address list.
if ccb.curBalancerName != grpclbName {
var addrs []resolver.Address
for _, addr := range ccs.ResolverState.Addresses {
if addr.Type == resolver.GRPCLB {
continue
}
addrs = append(addrs, addr)
}
ccs.ResolverState.Addresses = addrs
}
errCh <- ccb.balancer.UpdateClientConnState(*ccs)
})
if !ok {
@ -139,7 +125,9 @@ func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnStat
func (ccb *ccBalancerWrapper) updateSubConnState(sc balancer.SubConn, s connectivity.State, err error) {
ccb.mu.Lock()
ccb.serializer.Schedule(func(_ context.Context) {
ccb.balancer.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: s, ConnectionError: err})
// Even though it is optional for balancers, gracefulswitch ensures
// opts.StateListener is set, so this cannot ever be nil.
sc.(*acBalancerWrapper).stateListener(balancer.SubConnState{ConnectivityState: s, ConnectionError: err})
})
ccb.mu.Unlock()
}
@ -221,7 +209,7 @@ func (ccb *ccBalancerWrapper) closeBalancer(m ccbMode) {
}
ccb.mode = m
done := ccb.serializer.Done
done := ccb.serializer.Done()
b := ccb.balancer
ok := ccb.serializer.Schedule(func(_ context.Context) {
// Close the serializer to ensure that no more calls from gRPC are sent
@ -238,11 +226,9 @@ func (ccb *ccBalancerWrapper) closeBalancer(m ccbMode) {
}
ccb.mu.Unlock()
// Give enqueued callbacks a chance to finish.
// Give enqueued callbacks a chance to finish before closing the balancer.
<-done
// Spawn a goroutine to close the balancer (since it may block trying to
// cleanup all allocated resources) and return early.
go b.Close()
b.Close()
}
// exitIdleMode is invoked by grpc when the channel exits idle mode either
@ -314,29 +300,19 @@ func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer
channelz.Warningf(logger, ccb.cc.channelzID, "acBalancerWrapper: NewSubConn: failed to newAddrConn: %v", err)
return nil, err
}
acbw := &acBalancerWrapper{ac: ac, producers: make(map[balancer.ProducerBuilder]*refCountedProducer)}
acbw := &acBalancerWrapper{
ccb: ccb,
ac: ac,
producers: make(map[balancer.ProducerBuilder]*refCountedProducer),
stateListener: opts.StateListener,
}
ac.acbw = acbw
return acbw, nil
}
func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) {
if ccb.isIdleOrClosed() {
// It it safe to ignore this call when the balancer is closed or in idle
// because the ClientConn takes care of closing the connections.
//
// Not returning early from here when the balancer is closed or in idle
// leads to a deadlock though, because of the following sequence of
// calls when holding cc.mu:
// cc.exitIdleMode --> ccb.enterIdleMode --> gsw.Close -->
// ccb.RemoveAddrConn --> cc.removeAddrConn
return
}
acbw, ok := sc.(*acBalancerWrapper)
if !ok {
return
}
ccb.cc.removeAddrConn(acbw.ac, errConnDrain)
// The graceful switch balancer will never call this.
logger.Errorf("ccb RemoveSubConn(%v) called unexpectedly, sc")
}
func (ccb *ccBalancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {
@ -380,7 +356,9 @@ func (ccb *ccBalancerWrapper) Target() string {
// acBalancerWrapper is a wrapper on top of ac for balancers.
// It implements balancer.SubConn interface.
type acBalancerWrapper struct {
ac *addrConn // read-only
ac *addrConn // read-only
ccb *ccBalancerWrapper // read-only
stateListener func(balancer.SubConnState)
mu sync.Mutex
producers map[balancer.ProducerBuilder]*refCountedProducer
@ -398,6 +376,23 @@ func (acbw *acBalancerWrapper) Connect() {
go acbw.ac.connect()
}
func (acbw *acBalancerWrapper) Shutdown() {
ccb := acbw.ccb
if ccb.isIdleOrClosed() {
// It it safe to ignore this call when the balancer is closed or in idle
// because the ClientConn takes care of closing the connections.
//
// Not returning early from here when the balancer is closed or in idle
// leads to a deadlock though, because of the following sequence of
// calls when holding cc.mu:
// cc.exitIdleMode --> ccb.enterIdleMode --> gsw.Close -->
// ccb.RemoveAddrConn --> cc.removeAddrConn
return
}
ccb.cc.removeAddrConn(acbw.ac, errConnDrain)
}
// NewStream begins a streaming RPC on the addrConn. If the addrConn is not
// ready, blocks until it is or ctx expires. Returns an error when the context
// expires or the addrConn is shut down.
@ -411,7 +406,7 @@ func (acbw *acBalancerWrapper) NewStream(ctx context.Context, desc *StreamDesc,
// Invoke performs a unary RPC. If the addrConn is not ready, returns
// errSubConnNotReady.
func (acbw *acBalancerWrapper) Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...CallOption) error {
func (acbw *acBalancerWrapper) Invoke(ctx context.Context, method string, args any, reply any, opts ...CallOption) error {
cs, err := acbw.NewStream(ctx, unaryStreamDesc, method, opts...)
if err != nil {
return err

View File

@ -18,7 +18,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v4.22.0
// source: grpc/binlog/v1/binarylog.proto

View File

@ -26,12 +26,7 @@ import (
// received. This is typically called by generated code.
//
// All errors returned by Invoke are compatible with the status package.
func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply interface{}, opts ...CallOption) error {
if err := cc.idlenessMgr.onCallBegin(); err != nil {
return err
}
defer cc.idlenessMgr.onCallEnd()
func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply any, opts ...CallOption) error {
// allow interceptor to see all applicable call options, which means those
// configured as defaults from dial option as well as per-call options
opts = combine(cc.dopts.callOptions, opts)
@ -61,13 +56,13 @@ func combine(o1 []CallOption, o2 []CallOption) []CallOption {
// received. This is typically called by generated code.
//
// DEPRECATED: Use ClientConn.Invoke instead.
func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error {
func Invoke(ctx context.Context, method string, args, reply any, cc *ClientConn, opts ...CallOption) error {
return cc.Invoke(ctx, method, args, reply, opts...)
}
var unaryStreamDesc = &StreamDesc{ServerStreams: false, ClientStreams: false}
func invoke(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error {
func invoke(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error {
cs, err := newClientStream(ctx, unaryStreamDesc, cc, method, opts...)
if err != nil {
return err

View File

@ -34,9 +34,11 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/backoff"
"google.golang.org/grpc/internal/channelz"
"google.golang.org/grpc/internal/grpcsync"
"google.golang.org/grpc/internal/idle"
"google.golang.org/grpc/internal/pretty"
iresolver "google.golang.org/grpc/internal/resolver"
"google.golang.org/grpc/internal/transport"
@ -54,8 +56,6 @@ import (
const (
// minimum time to give a connection to complete
minConnectTimeout = 20 * time.Second
// must match grpclbName in grpclb/grpclb.go
grpclbName = "grpclb"
)
var (
@ -138,7 +138,6 @@ func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*ires
func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) {
cc := &ClientConn{
target: target,
csMgr: &connectivityStateManager{},
conns: make(map[*addrConn]struct{}),
dopts: defaultDialOptions(),
czData: new(channelzData),
@ -191,6 +190,8 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
// Register ClientConn with channelz.
cc.channelzRegistration(target)
cc.csMgr = newConnectivityStateManager(cc.ctx, cc.channelzID)
if err := cc.validateTransportCredentials(); err != nil {
return nil, err
}
@ -266,7 +267,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
// Configure idleness support with configured idle timeout or default idle
// timeout duration. Idleness can be explicitly disabled by the user, by
// setting the dial option to 0.
cc.idlenessMgr = newIdlenessManager(cc, cc.dopts.idleTimeout)
cc.idlenessMgr = idle.NewManager(idle.ManagerOptions{Enforcer: (*idler)(cc), Timeout: cc.dopts.idleTimeout, Logger: logger})
// Return early for non-blocking dials.
if !cc.dopts.block {
@ -317,6 +318,16 @@ func (cc *ClientConn) addTraceEvent(msg string) {
channelz.AddTraceEvent(logger, cc.channelzID, 0, ted)
}
type idler ClientConn
func (i *idler) EnterIdleMode() error {
return (*ClientConn)(i).enterIdleMode()
}
func (i *idler) ExitIdleMode() error {
return (*ClientConn)(i).exitIdleMode()
}
// exitIdleMode moves the channel out of idle mode by recreating the name
// resolver and load balancer.
func (cc *ClientConn) exitIdleMode() error {
@ -327,7 +338,7 @@ func (cc *ClientConn) exitIdleMode() error {
}
if cc.idlenessState != ccIdlenessStateIdle {
cc.mu.Unlock()
logger.Info("ClientConn asked to exit idle mode when not in idle mode")
channelz.Infof(logger, cc.channelzID, "ClientConn asked to exit idle mode, current mode is %v", cc.idlenessState)
return nil
}
@ -350,7 +361,7 @@ func (cc *ClientConn) exitIdleMode() error {
cc.idlenessState = ccIdlenessStateExitingIdle
exitedIdle := false
if cc.blockingpicker == nil {
cc.blockingpicker = newPickerWrapper()
cc.blockingpicker = newPickerWrapper(cc.dopts.copts.StatsHandlers)
} else {
cc.blockingpicker.exitIdleMode()
exitedIdle = true
@ -398,7 +409,8 @@ func (cc *ClientConn) enterIdleMode() error {
return ErrClientConnClosing
}
if cc.idlenessState != ccIdlenessStateActive {
logger.Error("ClientConn asked to enter idle mode when not active")
channelz.Errorf(logger, cc.channelzID, "ClientConn asked to enter idle mode, current mode is %v", cc.idlenessState)
cc.mu.Unlock()
return nil
}
@ -475,7 +487,6 @@ func (cc *ClientConn) validateTransportCredentials() error {
func (cc *ClientConn) channelzRegistration(target string) {
cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target)
cc.addTraceEvent("created")
cc.csMgr.channelzID = cc.channelzID
}
// chainUnaryClientInterceptors chains all unary client interceptors into one.
@ -492,7 +503,7 @@ func chainUnaryClientInterceptors(cc *ClientConn) {
} else if len(interceptors) == 1 {
chainedInt = interceptors[0]
} else {
chainedInt = func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error {
chainedInt = func(ctx context.Context, method string, req, reply any, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error {
return interceptors[0](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, 0, invoker), opts...)
}
}
@ -504,7 +515,7 @@ func getChainUnaryInvoker(interceptors []UnaryClientInterceptor, curr int, final
if curr == len(interceptors)-1 {
return finalInvoker
}
return func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error {
return func(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error {
return interceptors[curr+1](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, curr+1, finalInvoker), opts...)
}
}
@ -540,13 +551,27 @@ func getChainStreamer(interceptors []StreamClientInterceptor, curr int, finalStr
}
}
// newConnectivityStateManager creates an connectivityStateManager with
// the specified id.
func newConnectivityStateManager(ctx context.Context, id *channelz.Identifier) *connectivityStateManager {
return &connectivityStateManager{
channelzID: id,
pubSub: grpcsync.NewPubSub(ctx),
}
}
// connectivityStateManager keeps the connectivity.State of ClientConn.
// This struct will eventually be exported so the balancers can access it.
//
// TODO: If possible, get rid of the `connectivityStateManager` type, and
// provide this functionality using the `PubSub`, to avoid keeping track of
// the connectivity state at two places.
type connectivityStateManager struct {
mu sync.Mutex
state connectivity.State
notifyChan chan struct{}
channelzID *channelz.Identifier
pubSub *grpcsync.PubSub
}
// updateState updates the connectivity.State of ClientConn.
@ -562,6 +587,8 @@ func (csm *connectivityStateManager) updateState(state connectivity.State) {
return
}
csm.state = state
csm.pubSub.Publish(state)
channelz.Infof(logger, csm.channelzID, "Channel Connectivity change to %v", state)
if csm.notifyChan != nil {
// There are other goroutines waiting on this channel.
@ -591,7 +618,7 @@ func (csm *connectivityStateManager) getNotifyChan() <-chan struct{} {
type ClientConnInterface interface {
// Invoke performs a unary RPC and returns after the response is received
// into reply.
Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...CallOption) error
Invoke(ctx context.Context, method string, args any, reply any, opts ...CallOption) error
// NewStream begins a streaming RPC.
NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error)
}
@ -623,7 +650,7 @@ type ClientConn struct {
channelzID *channelz.Identifier // Channelz identifier for the channel.
resolverBuilder resolver.Builder // See parseTargetAndFindResolver().
balancerWrapper *ccBalancerWrapper // Uses gracefulswitch.balancer underneath.
idlenessMgr idlenessManager
idlenessMgr idle.Manager
// The following provide their own synchronization, and therefore don't
// require cc.mu to be held to access them.
@ -669,6 +696,19 @@ const (
ccIdlenessStateExitingIdle
)
func (s ccIdlenessState) String() string {
switch s {
case ccIdlenessStateActive:
return "active"
case ccIdlenessStateIdle:
return "idle"
case ccIdlenessStateExitingIdle:
return "exitingIdle"
default:
return "unknown"
}
}
// WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or
// ctx expires. A true value is returned in former case and false in latter.
//
@ -760,6 +800,10 @@ func init() {
panic(fmt.Sprintf("impossible error parsing empty service config: %v", cfg.Err))
}
emptyServiceConfig = cfg.Config.(*ServiceConfig)
internal.SubscribeToConnectivityStateChanges = func(cc *ClientConn, s grpcsync.Subscriber) func() {
return cc.csMgr.pubSub.Subscribe(s)
}
}
func (cc *ClientConn) maybeApplyDefaultServiceConfig(addrs []resolver.Address) {
@ -1047,8 +1091,8 @@ func (ac *addrConn) updateAddrs(addrs []resolver.Address) {
ac.cancel()
ac.ctx, ac.cancel = context.WithCancel(ac.cc.ctx)
// We have to defer here because GracefulClose => Close => onClose, which
// requires locking ac.mu.
// We have to defer here because GracefulClose => onClose, which requires
// locking ac.mu.
if ac.transport != nil {
defer ac.transport.GracefulClose()
ac.transport = nil
@ -1153,23 +1197,13 @@ func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, configSel
}
var newBalancerName string
if cc.sc != nil && cc.sc.lbConfig != nil {
if cc.sc == nil || (cc.sc.lbConfig == nil && cc.sc.LB == nil) {
// No service config or no LB policy specified in config.
newBalancerName = PickFirstBalancerName
} else if cc.sc.lbConfig != nil {
newBalancerName = cc.sc.lbConfig.name
} else {
var isGRPCLB bool
for _, a := range addrs {
if a.Type == resolver.GRPCLB {
isGRPCLB = true
break
}
}
if isGRPCLB {
newBalancerName = grpclbName
} else if cc.sc != nil && cc.sc.LB != nil {
newBalancerName = *cc.sc.LB
} else {
newBalancerName = PickFirstBalancerName
}
} else { // cc.sc.LB != nil
newBalancerName = *cc.sc.LB
}
cc.balancerWrapper.switchTo(newBalancerName)
}
@ -1208,7 +1242,10 @@ func (cc *ClientConn) ResetConnectBackoff() {
// Close tears down the ClientConn and all underlying connections.
func (cc *ClientConn) Close() error {
defer cc.cancel()
defer func() {
cc.cancel()
<-cc.csMgr.pubSub.Done()
}()
cc.mu.Lock()
if cc.conns == nil {
@ -1242,7 +1279,7 @@ func (cc *ClientConn) Close() error {
rWrapper.close()
}
if idlenessMgr != nil {
idlenessMgr.close()
idlenessMgr.Close()
}
for ac := range conns {
@ -1352,12 +1389,14 @@ func (ac *addrConn) resetTransport() {
if err := ac.tryAllAddrs(acCtx, addrs, connectDeadline); err != nil {
ac.cc.resolveNow(resolver.ResolveNowOptions{})
// After exhausting all addresses, the addrConn enters
// TRANSIENT_FAILURE.
ac.mu.Lock()
if acCtx.Err() != nil {
// addrConn was torn down.
ac.mu.Unlock()
return
}
ac.mu.Lock()
// After exhausting all addresses, the addrConn enters
// TRANSIENT_FAILURE.
ac.updateConnectivityState(connectivity.TransientFailure, err)
// Backoff.
@ -1553,7 +1592,7 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) {
// Set up the health check helper functions.
currentTr := ac.transport
newStream := func(method string) (interface{}, error) {
newStream := func(method string) (any, error) {
ac.mu.Lock()
if ac.transport != currentTr {
ac.mu.Unlock()
@ -1641,16 +1680,7 @@ func (ac *addrConn) tearDown(err error) {
ac.updateConnectivityState(connectivity.Shutdown, nil)
ac.cancel()
ac.curAddr = resolver.Address{}
if err == errConnDrain && curTr != nil {
// GracefulClose(...) may be executed multiple times when
// i) receiving multiple GoAway frames from the server; or
// ii) there are concurrent name resolver/Balancer triggered
// address removal and GoAway.
// We have to unlock and re-lock here because GracefulClose => Close => onClose, which requires locking ac.mu.
ac.mu.Unlock()
curTr.GracefulClose()
ac.mu.Lock()
}
channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{
Desc: "Subchannel deleted",
Severity: channelz.CtInfo,
@ -1664,6 +1694,29 @@ func (ac *addrConn) tearDown(err error) {
// being deleted right away.
channelz.RemoveEntry(ac.channelzID)
ac.mu.Unlock()
// We have to release the lock before the call to GracefulClose/Close here
// because both of them call onClose(), which requires locking ac.mu.
if curTr != nil {
if err == errConnDrain {
// Close the transport gracefully when the subConn is being shutdown.
//
// GracefulClose() may be executed multiple times if:
// - multiple GoAway frames are received from the server
// - there are concurrent name resolver or balancer triggered
// address removal and GoAway
curTr.GracefulClose()
} else {
// Hard close the transport when the channel is entering idle or is
// being shutdown. In the case where the channel is being shutdown,
// closing of transports is also taken care of by cancelation of cc.ctx.
// But in the case where the channel is entering idle, we need to
// explicitly close the transports here. Instead of distinguishing
// between these two cases, it is simpler to close the transport
// unconditionally here.
curTr.Close(err)
}
}
}
func (ac *addrConn) getState() connectivity.State {

View File

@ -27,8 +27,8 @@ import (
// omits the name/string, which vary between the two and are not needed for
// anything besides the registry in the encoding package.
type baseCodec interface {
Marshal(v interface{}) ([]byte, error)
Unmarshal(data []byte, v interface{}) error
Marshal(v any) ([]byte, error)
Unmarshal(data []byte, v any) error
}
var _ baseCodec = Codec(nil)
@ -41,9 +41,9 @@ var _ baseCodec = encoding.Codec(nil)
// Deprecated: use encoding.Codec instead.
type Codec interface {
// Marshal returns the wire format of v.
Marshal(v interface{}) ([]byte, error)
Marshal(v any) ([]byte, error)
// Unmarshal parses the wire format into v.
Unmarshal(data []byte, v interface{}) error
Unmarshal(data []byte, v any) error
// String returns the name of the Codec implementation. This is unused by
// gRPC.
String() string

View File

@ -139,6 +139,20 @@ func newJoinDialOption(opts ...DialOption) DialOption {
return &joinDialOption{opts: opts}
}
// WithSharedWriteBuffer allows reusing per-connection transport write buffer.
// If this option is set to true every connection will release the buffer after
// flushing the data on the wire.
//
// # Experimental
//
// Notice: This API is EXPERIMENTAL and may be changed or removed in a
// later release.
func WithSharedWriteBuffer(val bool) DialOption {
return newFuncDialOption(func(o *dialOptions) {
o.copts.SharedWriteBuffer = val
})
}
// WithWriteBufferSize determines how much data can be batched before doing a
// write on the wire. The corresponding memory allocation for this buffer will
// be twice the size to keep syscalls low. The default value for this buffer is

View File

@ -90,9 +90,9 @@ func GetCompressor(name string) Compressor {
// methods can be called from concurrent goroutines.
type Codec interface {
// Marshal returns the wire format of v.
Marshal(v interface{}) ([]byte, error)
Marshal(v any) ([]byte, error)
// Unmarshal parses the wire format into v.
Unmarshal(data []byte, v interface{}) error
Unmarshal(data []byte, v any) error
// Name returns the name of the Codec implementation. The returned string
// will be used as part of content type in transmission. The result must be
// static; the result cannot change between calls.

View File

@ -37,7 +37,7 @@ func init() {
// codec is a Codec implementation with protobuf. It is the default codec for gRPC.
type codec struct{}
func (codec) Marshal(v interface{}) ([]byte, error) {
func (codec) Marshal(v any) ([]byte, error) {
vv, ok := v.(proto.Message)
if !ok {
return nil, fmt.Errorf("failed to marshal, message is %T, want proto.Message", v)
@ -45,7 +45,7 @@ func (codec) Marshal(v interface{}) ([]byte, error) {
return proto.Marshal(vv)
}
func (codec) Unmarshal(data []byte, v interface{}) error {
func (codec) Unmarshal(data []byte, v any) error {
vv, ok := v.(proto.Message)
if !ok {
return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v)

View File

@ -31,71 +31,71 @@ type componentData struct {
var cache = map[string]*componentData{}
func (c *componentData) InfoDepth(depth int, args ...interface{}) {
args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
func (c *componentData) InfoDepth(depth int, args ...any) {
args = append([]any{"[" + string(c.name) + "]"}, args...)
grpclog.InfoDepth(depth+1, args...)
}
func (c *componentData) WarningDepth(depth int, args ...interface{}) {
args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
func (c *componentData) WarningDepth(depth int, args ...any) {
args = append([]any{"[" + string(c.name) + "]"}, args...)
grpclog.WarningDepth(depth+1, args...)
}
func (c *componentData) ErrorDepth(depth int, args ...interface{}) {
args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
func (c *componentData) ErrorDepth(depth int, args ...any) {
args = append([]any{"[" + string(c.name) + "]"}, args...)
grpclog.ErrorDepth(depth+1, args...)
}
func (c *componentData) FatalDepth(depth int, args ...interface{}) {
args = append([]interface{}{"[" + string(c.name) + "]"}, args...)
func (c *componentData) FatalDepth(depth int, args ...any) {
args = append([]any{"[" + string(c.name) + "]"}, args...)
grpclog.FatalDepth(depth+1, args...)
}
func (c *componentData) Info(args ...interface{}) {
func (c *componentData) Info(args ...any) {
c.InfoDepth(1, args...)
}
func (c *componentData) Warning(args ...interface{}) {
func (c *componentData) Warning(args ...any) {
c.WarningDepth(1, args...)
}
func (c *componentData) Error(args ...interface{}) {
func (c *componentData) Error(args ...any) {
c.ErrorDepth(1, args...)
}
func (c *componentData) Fatal(args ...interface{}) {
func (c *componentData) Fatal(args ...any) {
c.FatalDepth(1, args...)
}
func (c *componentData) Infof(format string, args ...interface{}) {
func (c *componentData) Infof(format string, args ...any) {
c.InfoDepth(1, fmt.Sprintf(format, args...))
}
func (c *componentData) Warningf(format string, args ...interface{}) {
func (c *componentData) Warningf(format string, args ...any) {
c.WarningDepth(1, fmt.Sprintf(format, args...))
}
func (c *componentData) Errorf(format string, args ...interface{}) {
func (c *componentData) Errorf(format string, args ...any) {
c.ErrorDepth(1, fmt.Sprintf(format, args...))
}
func (c *componentData) Fatalf(format string, args ...interface{}) {
func (c *componentData) Fatalf(format string, args ...any) {
c.FatalDepth(1, fmt.Sprintf(format, args...))
}
func (c *componentData) Infoln(args ...interface{}) {
func (c *componentData) Infoln(args ...any) {
c.InfoDepth(1, args...)
}
func (c *componentData) Warningln(args ...interface{}) {
func (c *componentData) Warningln(args ...any) {
c.WarningDepth(1, args...)
}
func (c *componentData) Errorln(args ...interface{}) {
func (c *componentData) Errorln(args ...any) {
c.ErrorDepth(1, args...)
}
func (c *componentData) Fatalln(args ...interface{}) {
func (c *componentData) Fatalln(args ...any) {
c.FatalDepth(1, args...)
}

View File

@ -42,53 +42,53 @@ func V(l int) bool {
}
// Info logs to the INFO log.
func Info(args ...interface{}) {
func Info(args ...any) {
grpclog.Logger.Info(args...)
}
// Infof logs to the INFO log. Arguments are handled in the manner of fmt.Printf.
func Infof(format string, args ...interface{}) {
func Infof(format string, args ...any) {
grpclog.Logger.Infof(format, args...)
}
// Infoln logs to the INFO log. Arguments are handled in the manner of fmt.Println.
func Infoln(args ...interface{}) {
func Infoln(args ...any) {
grpclog.Logger.Infoln(args...)
}
// Warning logs to the WARNING log.
func Warning(args ...interface{}) {
func Warning(args ...any) {
grpclog.Logger.Warning(args...)
}
// Warningf logs to the WARNING log. Arguments are handled in the manner of fmt.Printf.
func Warningf(format string, args ...interface{}) {
func Warningf(format string, args ...any) {
grpclog.Logger.Warningf(format, args...)
}
// Warningln logs to the WARNING log. Arguments are handled in the manner of fmt.Println.
func Warningln(args ...interface{}) {
func Warningln(args ...any) {
grpclog.Logger.Warningln(args...)
}
// Error logs to the ERROR log.
func Error(args ...interface{}) {
func Error(args ...any) {
grpclog.Logger.Error(args...)
}
// Errorf logs to the ERROR log. Arguments are handled in the manner of fmt.Printf.
func Errorf(format string, args ...interface{}) {
func Errorf(format string, args ...any) {
grpclog.Logger.Errorf(format, args...)
}
// Errorln logs to the ERROR log. Arguments are handled in the manner of fmt.Println.
func Errorln(args ...interface{}) {
func Errorln(args ...any) {
grpclog.Logger.Errorln(args...)
}
// Fatal logs to the FATAL log. Arguments are handled in the manner of fmt.Print.
// It calls os.Exit() with exit code 1.
func Fatal(args ...interface{}) {
func Fatal(args ...any) {
grpclog.Logger.Fatal(args...)
// Make sure fatal logs will exit.
os.Exit(1)
@ -96,7 +96,7 @@ func Fatal(args ...interface{}) {
// Fatalf logs to the FATAL log. Arguments are handled in the manner of fmt.Printf.
// It calls os.Exit() with exit code 1.
func Fatalf(format string, args ...interface{}) {
func Fatalf(format string, args ...any) {
grpclog.Logger.Fatalf(format, args...)
// Make sure fatal logs will exit.
os.Exit(1)
@ -104,7 +104,7 @@ func Fatalf(format string, args ...interface{}) {
// Fatalln logs to the FATAL log. Arguments are handled in the manner of fmt.Println.
// It calle os.Exit()) with exit code 1.
func Fatalln(args ...interface{}) {
func Fatalln(args ...any) {
grpclog.Logger.Fatalln(args...)
// Make sure fatal logs will exit.
os.Exit(1)
@ -113,20 +113,20 @@ func Fatalln(args ...interface{}) {
// Print prints to the logger. Arguments are handled in the manner of fmt.Print.
//
// Deprecated: use Info.
func Print(args ...interface{}) {
func Print(args ...any) {
grpclog.Logger.Info(args...)
}
// Printf prints to the logger. Arguments are handled in the manner of fmt.Printf.
//
// Deprecated: use Infof.
func Printf(format string, args ...interface{}) {
func Printf(format string, args ...any) {
grpclog.Logger.Infof(format, args...)
}
// Println prints to the logger. Arguments are handled in the manner of fmt.Println.
//
// Deprecated: use Infoln.
func Println(args ...interface{}) {
func Println(args ...any) {
grpclog.Logger.Infoln(args...)
}

View File

@ -24,12 +24,12 @@ import "google.golang.org/grpc/internal/grpclog"
//
// Deprecated: use LoggerV2.
type Logger interface {
Fatal(args ...interface{})
Fatalf(format string, args ...interface{})
Fatalln(args ...interface{})
Print(args ...interface{})
Printf(format string, args ...interface{})
Println(args ...interface{})
Fatal(args ...any)
Fatalf(format string, args ...any)
Fatalln(args ...any)
Print(args ...any)
Printf(format string, args ...any)
Println(args ...any)
}
// SetLogger sets the logger that is used in grpc. Call only from
@ -45,39 +45,39 @@ type loggerWrapper struct {
Logger
}
func (g *loggerWrapper) Info(args ...interface{}) {
func (g *loggerWrapper) Info(args ...any) {
g.Logger.Print(args...)
}
func (g *loggerWrapper) Infoln(args ...interface{}) {
func (g *loggerWrapper) Infoln(args ...any) {
g.Logger.Println(args...)
}
func (g *loggerWrapper) Infof(format string, args ...interface{}) {
func (g *loggerWrapper) Infof(format string, args ...any) {
g.Logger.Printf(format, args...)
}
func (g *loggerWrapper) Warning(args ...interface{}) {
func (g *loggerWrapper) Warning(args ...any) {
g.Logger.Print(args...)
}
func (g *loggerWrapper) Warningln(args ...interface{}) {
func (g *loggerWrapper) Warningln(args ...any) {
g.Logger.Println(args...)
}
func (g *loggerWrapper) Warningf(format string, args ...interface{}) {
func (g *loggerWrapper) Warningf(format string, args ...any) {
g.Logger.Printf(format, args...)
}
func (g *loggerWrapper) Error(args ...interface{}) {
func (g *loggerWrapper) Error(args ...any) {
g.Logger.Print(args...)
}
func (g *loggerWrapper) Errorln(args ...interface{}) {
func (g *loggerWrapper) Errorln(args ...any) {
g.Logger.Println(args...)
}
func (g *loggerWrapper) Errorf(format string, args ...interface{}) {
func (g *loggerWrapper) Errorf(format string, args ...any) {
g.Logger.Printf(format, args...)
}

View File

@ -33,35 +33,35 @@ import (
// LoggerV2 does underlying logging work for grpclog.
type LoggerV2 interface {
// Info logs to INFO log. Arguments are handled in the manner of fmt.Print.
Info(args ...interface{})
Info(args ...any)
// Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println.
Infoln(args ...interface{})
Infoln(args ...any)
// Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf.
Infof(format string, args ...interface{})
Infof(format string, args ...any)
// Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print.
Warning(args ...interface{})
Warning(args ...any)
// Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println.
Warningln(args ...interface{})
Warningln(args ...any)
// Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf.
Warningf(format string, args ...interface{})
Warningf(format string, args ...any)
// Error logs to ERROR log. Arguments are handled in the manner of fmt.Print.
Error(args ...interface{})
Error(args ...any)
// Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println.
Errorln(args ...interface{})
Errorln(args ...any)
// Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf.
Errorf(format string, args ...interface{})
Errorf(format string, args ...any)
// Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print.
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
// Implementations may also call os.Exit() with a non-zero exit code.
Fatal(args ...interface{})
Fatal(args ...any)
// Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println.
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
// Implementations may also call os.Exit() with a non-zero exit code.
Fatalln(args ...interface{})
Fatalln(args ...any)
// Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf.
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
// Implementations may also call os.Exit() with a non-zero exit code.
Fatalf(format string, args ...interface{})
Fatalf(format string, args ...any)
// V reports whether verbosity level l is at least the requested verbose level.
V(l int) bool
}
@ -182,53 +182,53 @@ func (g *loggerT) output(severity int, s string) {
g.m[severity].Output(2, string(b))
}
func (g *loggerT) Info(args ...interface{}) {
func (g *loggerT) Info(args ...any) {
g.output(infoLog, fmt.Sprint(args...))
}
func (g *loggerT) Infoln(args ...interface{}) {
func (g *loggerT) Infoln(args ...any) {
g.output(infoLog, fmt.Sprintln(args...))
}
func (g *loggerT) Infof(format string, args ...interface{}) {
func (g *loggerT) Infof(format string, args ...any) {
g.output(infoLog, fmt.Sprintf(format, args...))
}
func (g *loggerT) Warning(args ...interface{}) {
func (g *loggerT) Warning(args ...any) {
g.output(warningLog, fmt.Sprint(args...))
}
func (g *loggerT) Warningln(args ...interface{}) {
func (g *loggerT) Warningln(args ...any) {
g.output(warningLog, fmt.Sprintln(args...))
}
func (g *loggerT) Warningf(format string, args ...interface{}) {
func (g *loggerT) Warningf(format string, args ...any) {
g.output(warningLog, fmt.Sprintf(format, args...))
}
func (g *loggerT) Error(args ...interface{}) {
func (g *loggerT) Error(args ...any) {
g.output(errorLog, fmt.Sprint(args...))
}
func (g *loggerT) Errorln(args ...interface{}) {
func (g *loggerT) Errorln(args ...any) {
g.output(errorLog, fmt.Sprintln(args...))
}
func (g *loggerT) Errorf(format string, args ...interface{}) {
func (g *loggerT) Errorf(format string, args ...any) {
g.output(errorLog, fmt.Sprintf(format, args...))
}
func (g *loggerT) Fatal(args ...interface{}) {
func (g *loggerT) Fatal(args ...any) {
g.output(fatalLog, fmt.Sprint(args...))
os.Exit(1)
}
func (g *loggerT) Fatalln(args ...interface{}) {
func (g *loggerT) Fatalln(args ...any) {
g.output(fatalLog, fmt.Sprintln(args...))
os.Exit(1)
}
func (g *loggerT) Fatalf(format string, args ...interface{}) {
func (g *loggerT) Fatalf(format string, args ...any) {
g.output(fatalLog, fmt.Sprintf(format, args...))
os.Exit(1)
}
@ -248,11 +248,11 @@ func (g *loggerT) V(l int) bool {
type DepthLoggerV2 interface {
LoggerV2
// InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Println.
InfoDepth(depth int, args ...interface{})
InfoDepth(depth int, args ...any)
// WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Println.
WarningDepth(depth int, args ...interface{})
WarningDepth(depth int, args ...any)
// ErrorDepth logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Println.
ErrorDepth(depth int, args ...interface{})
ErrorDepth(depth int, args ...any)
// FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Println.
FatalDepth(depth int, args ...interface{})
FatalDepth(depth int, args ...any)
}

View File

@ -23,7 +23,7 @@ import (
)
// UnaryInvoker is called by UnaryClientInterceptor to complete RPCs.
type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error
type UnaryInvoker func(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error
// UnaryClientInterceptor intercepts the execution of a unary RPC on the client.
// Unary interceptors can be specified as a DialOption, using
@ -40,7 +40,7 @@ type UnaryInvoker func(ctx context.Context, method string, req, reply interface{
// defaults from the ClientConn as well as per-call options.
//
// The returned error must be compatible with the status package.
type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error
type UnaryClientInterceptor func(ctx context.Context, method string, req, reply any, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error
// Streamer is called by StreamClientInterceptor to create a ClientStream.
type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error)
@ -66,7 +66,7 @@ type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *Cli
// server side. All per-rpc information may be mutated by the interceptor.
type UnaryServerInfo struct {
// Server is the service implementation the user provides. This is read-only.
Server interface{}
Server any
// FullMethod is the full RPC method string, i.e., /package.service/method.
FullMethod string
}
@ -78,13 +78,13 @@ type UnaryServerInfo struct {
// status package, or be one of the context errors. Otherwise, gRPC will use
// codes.Unknown as the status code and err.Error() as the status message of the
// RPC.
type UnaryHandler func(ctx context.Context, req interface{}) (interface{}, error)
type UnaryHandler func(ctx context.Context, req any) (any, error)
// UnaryServerInterceptor provides a hook to intercept the execution of a unary RPC on the server. info
// contains all the information of this RPC the interceptor can operate on. And handler is the wrapper
// of the service method implementation. It is the responsibility of the interceptor to invoke handler
// to complete the RPC.
type UnaryServerInterceptor func(ctx context.Context, req interface{}, info *UnaryServerInfo, handler UnaryHandler) (resp interface{}, err error)
type UnaryServerInterceptor func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (resp any, err error)
// StreamServerInfo consists of various information about a streaming RPC on
// server side. All per-rpc information may be mutated by the interceptor.
@ -101,4 +101,4 @@ type StreamServerInfo struct {
// info contains all the information of this RPC the interceptor can operate on. And handler is the
// service method implementation. It is the responsibility of the interceptor to invoke handler to
// complete the RPC.
type StreamServerInterceptor func(srv interface{}, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error
type StreamServerInterceptor func(srv any, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error

View File

@ -200,8 +200,8 @@ func (gsb *Balancer) ExitIdle() {
}
}
// UpdateSubConnState forwards the update to the appropriate child.
func (gsb *Balancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
// updateSubConnState forwards the update to the appropriate child.
func (gsb *Balancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState, cb func(balancer.SubConnState)) {
gsb.currentMu.Lock()
defer gsb.currentMu.Unlock()
gsb.mu.Lock()
@ -214,13 +214,26 @@ func (gsb *Balancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC
} else if gsb.balancerPending != nil && gsb.balancerPending.subconns[sc] {
balToUpdate = gsb.balancerPending
}
gsb.mu.Unlock()
if balToUpdate == nil {
// SubConn belonged to a stale lb policy that has not yet fully closed,
// or the balancer was already closed.
gsb.mu.Unlock()
return
}
balToUpdate.UpdateSubConnState(sc, state)
if state.ConnectivityState == connectivity.Shutdown {
delete(balToUpdate.subconns, sc)
}
gsb.mu.Unlock()
if cb != nil {
cb(state)
} else {
balToUpdate.UpdateSubConnState(sc, state)
}
}
// UpdateSubConnState forwards the update to the appropriate child.
func (gsb *Balancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
gsb.updateSubConnState(sc, state, nil)
}
// Close closes any active child balancers.
@ -242,7 +255,7 @@ func (gsb *Balancer) Close() {
//
// It implements the balancer.ClientConn interface and is passed down in that
// capacity to the wrapped balancer. It maintains a set of subConns created by
// the wrapped balancer and calls from the latter to create/update/remove
// the wrapped balancer and calls from the latter to create/update/shutdown
// SubConns update this set before being forwarded to the parent ClientConn.
// State updates from the wrapped balancer can result in invocation of the
// graceful switch logic.
@ -254,21 +267,10 @@ type balancerWrapper struct {
subconns map[balancer.SubConn]bool // subconns created by this balancer
}
func (bw *balancerWrapper) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
if state.ConnectivityState == connectivity.Shutdown {
bw.gsb.mu.Lock()
delete(bw.subconns, sc)
bw.gsb.mu.Unlock()
}
// There is no need to protect this read with a mutex, as the write to the
// Balancer field happens in SwitchTo, which completes before this can be
// called.
bw.Balancer.UpdateSubConnState(sc, state)
}
// Close closes the underlying LB policy and removes the subconns it created. bw
// must not be referenced via balancerCurrent or balancerPending in gsb when
// called. gsb.mu must not be held. Does not panic with a nil receiver.
// Close closes the underlying LB policy and shuts down the subconns it
// created. bw must not be referenced via balancerCurrent or balancerPending in
// gsb when called. gsb.mu must not be held. Does not panic with a nil
// receiver.
func (bw *balancerWrapper) Close() {
// before Close is called.
if bw == nil {
@ -281,7 +283,7 @@ func (bw *balancerWrapper) Close() {
bw.Balancer.Close()
bw.gsb.mu.Lock()
for sc := range bw.subconns {
bw.gsb.cc.RemoveSubConn(sc)
sc.Shutdown()
}
bw.gsb.mu.Unlock()
}
@ -335,13 +337,16 @@ func (bw *balancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.Ne
}
bw.gsb.mu.Unlock()
var sc balancer.SubConn
oldListener := opts.StateListener
opts.StateListener = func(state balancer.SubConnState) { bw.gsb.updateSubConnState(sc, state, oldListener) }
sc, err := bw.gsb.cc.NewSubConn(addrs, opts)
if err != nil {
return nil, err
}
bw.gsb.mu.Lock()
if !bw.gsb.balancerCurrentOrPending(bw) { // balancer was closed during this call
bw.gsb.cc.RemoveSubConn(sc)
sc.Shutdown()
bw.gsb.mu.Unlock()
return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw)
}
@ -360,13 +365,9 @@ func (bw *balancerWrapper) ResolveNow(opts resolver.ResolveNowOptions) {
}
func (bw *balancerWrapper) RemoveSubConn(sc balancer.SubConn) {
bw.gsb.mu.Lock()
if !bw.gsb.balancerCurrentOrPending(bw) {
bw.gsb.mu.Unlock()
return
}
bw.gsb.mu.Unlock()
bw.gsb.cc.RemoveSubConn(sc)
// Note: existing third party balancers may call this, so it must remain
// until RemoveSubConn is fully removed.
sc.Shutdown()
}
func (bw *balancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) {

View File

@ -25,7 +25,7 @@ import (
// Parser converts loads from metadata into a concrete type.
type Parser interface {
// Parse parses loads from metadata.
Parse(md metadata.MD) interface{}
Parse(md metadata.MD) any
}
var parser Parser
@ -38,7 +38,7 @@ func SetParser(lr Parser) {
}
// Parse calls parser.Read().
func Parse(md metadata.MD) interface{} {
func Parse(md metadata.MD) any {
if parser == nil {
return nil
}

View File

@ -230,7 +230,7 @@ type ClientMessage struct {
OnClientSide bool
// Message can be a proto.Message or []byte. Other messages formats are not
// supported.
Message interface{}
Message any
}
func (c *ClientMessage) toProto() *binlogpb.GrpcLogEntry {
@ -270,7 +270,7 @@ type ServerMessage struct {
OnClientSide bool
// Message can be a proto.Message or []byte. Other messages formats are not
// supported.
Message interface{}
Message any
}
func (c *ServerMessage) toProto() *binlogpb.GrpcLogEntry {

View File

@ -28,25 +28,25 @@ import "sync"
// the underlying mutex used for synchronization.
//
// Unbounded supports values of any type to be stored in it by using a channel
// of `interface{}`. This means that a call to Put() incurs an extra memory
// allocation, and also that users need a type assertion while reading. For
// performance critical code paths, using Unbounded is strongly discouraged and
// defining a new type specific implementation of this buffer is preferred. See
// of `any`. This means that a call to Put() incurs an extra memory allocation,
// and also that users need a type assertion while reading. For performance
// critical code paths, using Unbounded is strongly discouraged and defining a
// new type specific implementation of this buffer is preferred. See
// internal/transport/transport.go for an example of this.
type Unbounded struct {
c chan interface{}
c chan any
closed bool
mu sync.Mutex
backlog []interface{}
backlog []any
}
// NewUnbounded returns a new instance of Unbounded.
func NewUnbounded() *Unbounded {
return &Unbounded{c: make(chan interface{}, 1)}
return &Unbounded{c: make(chan any, 1)}
}
// Put adds t to the unbounded buffer.
func (b *Unbounded) Put(t interface{}) {
func (b *Unbounded) Put(t any) {
b.mu.Lock()
defer b.mu.Unlock()
if b.closed {
@ -89,7 +89,7 @@ func (b *Unbounded) Load() {
//
// If the unbounded buffer is closed, the read channel returned by this method
// is closed.
func (b *Unbounded) Get() <-chan interface{} {
func (b *Unbounded) Get() <-chan any {
return b.c
}

View File

@ -24,9 +24,7 @@
package channelz
import (
"context"
"errors"
"fmt"
"sort"
"sync"
"sync/atomic"
@ -40,8 +38,11 @@ const (
)
var (
db dbWrapper
idGen idGenerator
// IDGen is the global channelz entity ID generator. It should not be used
// outside this package except by tests.
IDGen IDGenerator
db dbWrapper
// EntryPerPage defines the number of channelz entries to be shown on a web page.
EntryPerPage = int64(50)
curState int32
@ -52,14 +53,14 @@ var (
func TurnOn() {
if !IsOn() {
db.set(newChannelMap())
idGen.reset()
IDGen.Reset()
atomic.StoreInt32(&curState, 1)
}
}
// IsOn returns whether channelz data collection is on.
func IsOn() bool {
return atomic.CompareAndSwapInt32(&curState, 1, 1)
return atomic.LoadInt32(&curState) == 1
}
// SetMaxTraceEntry sets maximum number of trace entry per entity (i.e. channel/subchannel).
@ -97,43 +98,6 @@ func (d *dbWrapper) get() *channelMap {
return d.DB
}
// NewChannelzStorageForTesting initializes channelz data storage and id
// generator for testing purposes.
//
// Returns a cleanup function to be invoked by the test, which waits for up to
// 10s for all channelz state to be reset by the grpc goroutines when those
// entities get closed. This cleanup function helps with ensuring that tests
// don't mess up each other.
func NewChannelzStorageForTesting() (cleanup func() error) {
db.set(newChannelMap())
idGen.reset()
return func() error {
cm := db.get()
if cm == nil {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
cm.mu.RLock()
topLevelChannels, servers, channels, subChannels, listenSockets, normalSockets := len(cm.topLevelChannels), len(cm.servers), len(cm.channels), len(cm.subChannels), len(cm.listenSockets), len(cm.normalSockets)
cm.mu.RUnlock()
if err := ctx.Err(); err != nil {
return fmt.Errorf("after 10s the channelz map has not been cleaned up yet, topchannels: %d, servers: %d, channels: %d, subchannels: %d, listen sockets: %d, normal sockets: %d", topLevelChannels, servers, channels, subChannels, listenSockets, normalSockets)
}
if topLevelChannels == 0 && servers == 0 && channels == 0 && subChannels == 0 && listenSockets == 0 && normalSockets == 0 {
return nil
}
<-ticker.C
}
}
}
// GetTopChannels returns a slice of top channel's ChannelMetric, along with a
// boolean indicating whether there's more top channels to be queried for.
//
@ -193,7 +157,7 @@ func GetServer(id int64) *ServerMetric {
//
// If channelz is not turned ON, the channelz database is not mutated.
func RegisterChannel(c Channel, pid *Identifier, ref string) *Identifier {
id := idGen.genID()
id := IDGen.genID()
var parent int64
isTopChannel := true
if pid != nil {
@ -229,7 +193,7 @@ func RegisterSubChannel(c Channel, pid *Identifier, ref string) (*Identifier, er
if pid == nil {
return nil, errors.New("a SubChannel's parent id cannot be nil")
}
id := idGen.genID()
id := IDGen.genID()
if !IsOn() {
return newIdentifer(RefSubChannel, id, pid), nil
}
@ -251,7 +215,7 @@ func RegisterSubChannel(c Channel, pid *Identifier, ref string) (*Identifier, er
//
// If channelz is not turned ON, the channelz database is not mutated.
func RegisterServer(s Server, ref string) *Identifier {
id := idGen.genID()
id := IDGen.genID()
if !IsOn() {
return newIdentifer(RefServer, id, nil)
}
@ -277,7 +241,7 @@ func RegisterListenSocket(s Socket, pid *Identifier, ref string) (*Identifier, e
if pid == nil {
return nil, errors.New("a ListenSocket's parent id cannot be 0")
}
id := idGen.genID()
id := IDGen.genID()
if !IsOn() {
return newIdentifer(RefListenSocket, id, pid), nil
}
@ -297,7 +261,7 @@ func RegisterNormalSocket(s Socket, pid *Identifier, ref string) (*Identifier, e
if pid == nil {
return nil, errors.New("a NormalSocket's parent id cannot be 0")
}
id := idGen.genID()
id := IDGen.genID()
if !IsOn() {
return newIdentifer(RefNormalSocket, id, pid), nil
}
@ -776,14 +740,17 @@ func (c *channelMap) GetServer(id int64) *ServerMetric {
return sm
}
type idGenerator struct {
// IDGenerator is an incrementing atomic that tracks IDs for channelz entities.
type IDGenerator struct {
id int64
}
func (i *idGenerator) reset() {
// Reset resets the generated ID back to zero. Should only be used at
// initialization or by tests sensitive to the ID number.
func (i *IDGenerator) Reset() {
atomic.StoreInt64(&i.id, 0)
}
func (i *idGenerator) genID() int64 {
func (i *IDGenerator) genID() int64 {
return atomic.AddInt64(&i.id, 1)
}

View File

@ -31,7 +31,7 @@ func withParens(id *Identifier) string {
}
// Info logs and adds a trace event if channelz is on.
func Info(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) {
func Info(l grpclog.DepthLoggerV2, id *Identifier, args ...any) {
AddTraceEvent(l, id, 1, &TraceEventDesc{
Desc: fmt.Sprint(args...),
Severity: CtInfo,
@ -39,7 +39,7 @@ func Info(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) {
}
// Infof logs and adds a trace event if channelz is on.
func Infof(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...interface{}) {
func Infof(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...any) {
AddTraceEvent(l, id, 1, &TraceEventDesc{
Desc: fmt.Sprintf(format, args...),
Severity: CtInfo,
@ -47,7 +47,7 @@ func Infof(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...inter
}
// Warning logs and adds a trace event if channelz is on.
func Warning(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) {
func Warning(l grpclog.DepthLoggerV2, id *Identifier, args ...any) {
AddTraceEvent(l, id, 1, &TraceEventDesc{
Desc: fmt.Sprint(args...),
Severity: CtWarning,
@ -55,7 +55,7 @@ func Warning(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) {
}
// Warningf logs and adds a trace event if channelz is on.
func Warningf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...interface{}) {
func Warningf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...any) {
AddTraceEvent(l, id, 1, &TraceEventDesc{
Desc: fmt.Sprintf(format, args...),
Severity: CtWarning,
@ -63,7 +63,7 @@ func Warningf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...in
}
// Error logs and adds a trace event if channelz is on.
func Error(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) {
func Error(l grpclog.DepthLoggerV2, id *Identifier, args ...any) {
AddTraceEvent(l, id, 1, &TraceEventDesc{
Desc: fmt.Sprint(args...),
Severity: CtError,
@ -71,7 +71,7 @@ func Error(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) {
}
// Errorf logs and adds a trace event if channelz is on.
func Errorf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...interface{}) {
func Errorf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...any) {
AddTraceEvent(l, id, 1, &TraceEventDesc{
Desc: fmt.Sprintf(format, args...),
Severity: CtError,

View File

@ -628,6 +628,7 @@ type tracedChannel interface {
type channelTrace struct {
cm *channelMap
clearCalled bool
createdTime time.Time
eventCount int64
mu sync.Mutex
@ -656,6 +657,10 @@ func (c *channelTrace) append(e *TraceEvent) {
}
func (c *channelTrace) clear() {
if c.clearCalled {
return
}
c.clearCalled = true
c.mu.Lock()
for _, e := range c.events {
if e.RefID != 0 {

View File

@ -23,7 +23,7 @@ import (
)
// GetSocketOption gets the socket option info of the conn.
func GetSocketOption(socket interface{}) *SocketOptionData {
func GetSocketOption(socket any) *SocketOptionData {
c, ok := socket.(syscall.Conn)
if !ok {
return nil

View File

@ -22,6 +22,6 @@
package channelz
// GetSocketOption gets the socket option info of the conn.
func GetSocketOption(c interface{}) *SocketOptionData {
func GetSocketOption(c any) *SocketOptionData {
return nil
}

View File

@ -25,12 +25,12 @@ import (
type requestInfoKey struct{}
// NewRequestInfoContext creates a context with ri.
func NewRequestInfoContext(ctx context.Context, ri interface{}) context.Context {
func NewRequestInfoContext(ctx context.Context, ri any) context.Context {
return context.WithValue(ctx, requestInfoKey{}, ri)
}
// RequestInfoFromContext extracts the RequestInfo from ctx.
func RequestInfoFromContext(ctx context.Context) interface{} {
func RequestInfoFromContext(ctx context.Context) any {
return ctx.Value(requestInfoKey{})
}
@ -39,11 +39,11 @@ func RequestInfoFromContext(ctx context.Context) interface{} {
type clientHandshakeInfoKey struct{}
// ClientHandshakeInfoFromContext extracts the ClientHandshakeInfo from ctx.
func ClientHandshakeInfoFromContext(ctx context.Context) interface{} {
func ClientHandshakeInfoFromContext(ctx context.Context) any {
return ctx.Value(clientHandshakeInfoKey{})
}
// NewClientHandshakeInfoContext creates a context with chi.
func NewClientHandshakeInfoContext(ctx context.Context, chi interface{}) context.Context {
func NewClientHandshakeInfoContext(ctx context.Context, chi any) context.Context {
return context.WithValue(ctx, clientHandshakeInfoKey{}, chi)
}

View File

@ -37,9 +37,12 @@ var (
// checking which NACKs configs specifying ring sizes > 8*1024*1024 (~8M).
RingHashCap = uint64FromEnv("GRPC_RING_HASH_CAP", 4096, 1, 8*1024*1024)
// PickFirstLBConfig is set if we should support configuration of the
// pick_first LB policy, which can be enabled by setting the environment
// variable "GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG" to "true".
PickFirstLBConfig = boolFromEnv("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", false)
// pick_first LB policy.
PickFirstLBConfig = boolFromEnv("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", true)
// LeastRequestLB is set if we should support the least_request_experimental
// LB policy, which can be enabled by setting the environment variable
// "GRPC_EXPERIMENTAL_ENABLE_LEAST_REQUEST" to "true".
LeastRequestLB = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_LEAST_REQUEST", false)
// ALTSMaxConcurrentHandshakes is the maximum number of concurrent ALTS
// handshakes that can be performed.
ALTSMaxConcurrentHandshakes = uint64FromEnv("GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES", 100, 1, 100)

View File

@ -30,7 +30,7 @@ var Logger LoggerV2
var DepthLogger DepthLoggerV2
// InfoDepth logs to the INFO log at the specified depth.
func InfoDepth(depth int, args ...interface{}) {
func InfoDepth(depth int, args ...any) {
if DepthLogger != nil {
DepthLogger.InfoDepth(depth, args...)
} else {
@ -39,7 +39,7 @@ func InfoDepth(depth int, args ...interface{}) {
}
// WarningDepth logs to the WARNING log at the specified depth.
func WarningDepth(depth int, args ...interface{}) {
func WarningDepth(depth int, args ...any) {
if DepthLogger != nil {
DepthLogger.WarningDepth(depth, args...)
} else {
@ -48,7 +48,7 @@ func WarningDepth(depth int, args ...interface{}) {
}
// ErrorDepth logs to the ERROR log at the specified depth.
func ErrorDepth(depth int, args ...interface{}) {
func ErrorDepth(depth int, args ...any) {
if DepthLogger != nil {
DepthLogger.ErrorDepth(depth, args...)
} else {
@ -57,7 +57,7 @@ func ErrorDepth(depth int, args ...interface{}) {
}
// FatalDepth logs to the FATAL log at the specified depth.
func FatalDepth(depth int, args ...interface{}) {
func FatalDepth(depth int, args ...any) {
if DepthLogger != nil {
DepthLogger.FatalDepth(depth, args...)
} else {
@ -71,35 +71,35 @@ func FatalDepth(depth int, args ...interface{}) {
// is defined here to avoid a circular dependency.
type LoggerV2 interface {
// Info logs to INFO log. Arguments are handled in the manner of fmt.Print.
Info(args ...interface{})
Info(args ...any)
// Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println.
Infoln(args ...interface{})
Infoln(args ...any)
// Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf.
Infof(format string, args ...interface{})
Infof(format string, args ...any)
// Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print.
Warning(args ...interface{})
Warning(args ...any)
// Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println.
Warningln(args ...interface{})
Warningln(args ...any)
// Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf.
Warningf(format string, args ...interface{})
Warningf(format string, args ...any)
// Error logs to ERROR log. Arguments are handled in the manner of fmt.Print.
Error(args ...interface{})
Error(args ...any)
// Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println.
Errorln(args ...interface{})
Errorln(args ...any)
// Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf.
Errorf(format string, args ...interface{})
Errorf(format string, args ...any)
// Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print.
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
// Implementations may also call os.Exit() with a non-zero exit code.
Fatal(args ...interface{})
Fatal(args ...any)
// Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println.
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
// Implementations may also call os.Exit() with a non-zero exit code.
Fatalln(args ...interface{})
Fatalln(args ...any)
// Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf.
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
// Implementations may also call os.Exit() with a non-zero exit code.
Fatalf(format string, args ...interface{})
Fatalf(format string, args ...any)
// V reports whether verbosity level l is at least the requested verbose level.
V(l int) bool
}
@ -116,11 +116,11 @@ type LoggerV2 interface {
// later release.
type DepthLoggerV2 interface {
// InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Println.
InfoDepth(depth int, args ...interface{})
InfoDepth(depth int, args ...any)
// WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Println.
WarningDepth(depth int, args ...interface{})
WarningDepth(depth int, args ...any)
// ErrorDepth logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Println.
ErrorDepth(depth int, args ...interface{})
ErrorDepth(depth int, args ...any)
// FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Println.
FatalDepth(depth int, args ...interface{})
FatalDepth(depth int, args ...any)
}

View File

@ -31,7 +31,7 @@ type PrefixLogger struct {
}
// Infof does info logging.
func (pl *PrefixLogger) Infof(format string, args ...interface{}) {
func (pl *PrefixLogger) Infof(format string, args ...any) {
if pl != nil {
// Handle nil, so the tests can pass in a nil logger.
format = pl.prefix + format
@ -42,7 +42,7 @@ func (pl *PrefixLogger) Infof(format string, args ...interface{}) {
}
// Warningf does warning logging.
func (pl *PrefixLogger) Warningf(format string, args ...interface{}) {
func (pl *PrefixLogger) Warningf(format string, args ...any) {
if pl != nil {
format = pl.prefix + format
pl.logger.WarningDepth(1, fmt.Sprintf(format, args...))
@ -52,7 +52,7 @@ func (pl *PrefixLogger) Warningf(format string, args ...interface{}) {
}
// Errorf does error logging.
func (pl *PrefixLogger) Errorf(format string, args ...interface{}) {
func (pl *PrefixLogger) Errorf(format string, args ...any) {
if pl != nil {
format = pl.prefix + format
pl.logger.ErrorDepth(1, fmt.Sprintf(format, args...))
@ -62,7 +62,7 @@ func (pl *PrefixLogger) Errorf(format string, args ...interface{}) {
}
// Debugf does info logging at verbose level 2.
func (pl *PrefixLogger) Debugf(format string, args ...interface{}) {
func (pl *PrefixLogger) Debugf(format string, args ...any) {
// TODO(6044): Refactor interfaces LoggerV2 and DepthLogger, and maybe
// rewrite PrefixLogger a little to ensure that we don't use the global
// `Logger` here, and instead use the `logger` field.

View File

@ -32,10 +32,10 @@ import (
//
// This type is safe for concurrent access.
type CallbackSerializer struct {
// Done is closed once the serializer is shut down completely, i.e all
// done is closed once the serializer is shut down completely, i.e all
// scheduled callbacks are executed and the serializer has deallocated all
// its resources.
Done chan struct{}
done chan struct{}
callbacks *buffer.Unbounded
closedMu sync.Mutex
@ -48,12 +48,12 @@ type CallbackSerializer struct {
// callbacks will be added once this context is canceled, and any pending un-run
// callbacks will be executed before the serializer is shut down.
func NewCallbackSerializer(ctx context.Context) *CallbackSerializer {
t := &CallbackSerializer{
Done: make(chan struct{}),
cs := &CallbackSerializer{
done: make(chan struct{}),
callbacks: buffer.NewUnbounded(),
}
go t.run(ctx)
return t
go cs.run(ctx)
return cs
}
// Schedule adds a callback to be scheduled after existing callbacks are run.
@ -64,56 +64,62 @@ func NewCallbackSerializer(ctx context.Context) *CallbackSerializer {
// Return value indicates if the callback was successfully added to the list of
// callbacks to be executed by the serializer. It is not possible to add
// callbacks once the context passed to NewCallbackSerializer is cancelled.
func (t *CallbackSerializer) Schedule(f func(ctx context.Context)) bool {
t.closedMu.Lock()
defer t.closedMu.Unlock()
func (cs *CallbackSerializer) Schedule(f func(ctx context.Context)) bool {
cs.closedMu.Lock()
defer cs.closedMu.Unlock()
if t.closed {
if cs.closed {
return false
}
t.callbacks.Put(f)
cs.callbacks.Put(f)
return true
}
func (t *CallbackSerializer) run(ctx context.Context) {
func (cs *CallbackSerializer) run(ctx context.Context) {
var backlog []func(context.Context)
defer close(t.Done)
defer close(cs.done)
for ctx.Err() == nil {
select {
case <-ctx.Done():
// Do nothing here. Next iteration of the for loop will not happen,
// since ctx.Err() would be non-nil.
case callback, ok := <-t.callbacks.Get():
case callback, ok := <-cs.callbacks.Get():
if !ok {
return
}
t.callbacks.Load()
cs.callbacks.Load()
callback.(func(ctx context.Context))(ctx)
}
}
// Fetch pending callbacks if any, and execute them before returning from
// this method and closing t.Done.
t.closedMu.Lock()
t.closed = true
backlog = t.fetchPendingCallbacks()
t.callbacks.Close()
t.closedMu.Unlock()
// this method and closing cs.done.
cs.closedMu.Lock()
cs.closed = true
backlog = cs.fetchPendingCallbacks()
cs.callbacks.Close()
cs.closedMu.Unlock()
for _, b := range backlog {
b(ctx)
}
}
func (t *CallbackSerializer) fetchPendingCallbacks() []func(context.Context) {
func (cs *CallbackSerializer) fetchPendingCallbacks() []func(context.Context) {
var backlog []func(context.Context)
for {
select {
case b := <-t.callbacks.Get():
case b := <-cs.callbacks.Get():
backlog = append(backlog, b.(func(context.Context)))
t.callbacks.Load()
cs.callbacks.Load()
default:
return backlog
}
}
}
// Done returns a channel that is closed after the context passed to
// NewCallbackSerializer is canceled and all callbacks have been executed.
func (cs *CallbackSerializer) Done() <-chan struct{} {
return cs.done
}

View File

@ -29,7 +29,7 @@ import (
type Subscriber interface {
// OnMessage is invoked when a new message is published. Implementations
// must not block in this method.
OnMessage(msg interface{})
OnMessage(msg any)
}
// PubSub is a simple one-to-many publish-subscribe system that supports
@ -40,25 +40,23 @@ type Subscriber interface {
// subscribers interested in receiving these messages register a callback
// via the Subscribe() method.
//
// Once a PubSub is stopped, no more messages can be published, and
// it is guaranteed that no more subscriber callback will be invoked.
// Once a PubSub is stopped, no more messages can be published, but any pending
// published messages will be delivered to the subscribers. Done may be used
// to determine when all published messages have been delivered.
type PubSub struct {
cs *CallbackSerializer
cancel context.CancelFunc
cs *CallbackSerializer
// Access to the below fields are guarded by this mutex.
mu sync.Mutex
msg interface{}
msg any
subscribers map[Subscriber]bool
stopped bool
}
// NewPubSub returns a new PubSub instance.
func NewPubSub() *PubSub {
ctx, cancel := context.WithCancel(context.Background())
// NewPubSub returns a new PubSub instance. Users should cancel the
// provided context to shutdown the PubSub.
func NewPubSub(ctx context.Context) *PubSub {
return &PubSub{
cs: NewCallbackSerializer(ctx),
cancel: cancel,
subscribers: map[Subscriber]bool{},
}
}
@ -75,10 +73,6 @@ func (ps *PubSub) Subscribe(sub Subscriber) (cancel func()) {
ps.mu.Lock()
defer ps.mu.Unlock()
if ps.stopped {
return func() {}
}
ps.subscribers[sub] = true
if ps.msg != nil {
@ -102,14 +96,10 @@ func (ps *PubSub) Subscribe(sub Subscriber) (cancel func()) {
// Publish publishes the provided message to the PubSub, and invokes
// callbacks registered by subscribers asynchronously.
func (ps *PubSub) Publish(msg interface{}) {
func (ps *PubSub) Publish(msg any) {
ps.mu.Lock()
defer ps.mu.Unlock()
if ps.stopped {
return
}
ps.msg = msg
for sub := range ps.subscribers {
s := sub
@ -124,13 +114,8 @@ func (ps *PubSub) Publish(msg interface{}) {
}
}
// Stop shuts down the PubSub and releases any resources allocated by it.
// It is guaranteed that no subscriber callbacks would be invoked once this
// method returns.
func (ps *PubSub) Stop() {
ps.mu.Lock()
defer ps.mu.Unlock()
ps.stopped = true
ps.cancel()
// Done returns a channel that is closed after the context passed to NewPubSub
// is canceled and all updates have been sent to subscribers.
func (ps *PubSub) Done() <-chan struct{} {
return ps.cs.Done()
}

View File

@ -16,7 +16,9 @@
*
*/
package grpc
// Package idle contains a component for managing idleness (entering and exiting)
// based on RPC activity.
package idle
import (
"fmt"
@ -24,6 +26,8 @@ import (
"sync"
"sync/atomic"
"time"
"google.golang.org/grpc/grpclog"
)
// For overriding in unit tests.
@ -31,31 +35,31 @@ var timeAfterFunc = func(d time.Duration, f func()) *time.Timer {
return time.AfterFunc(d, f)
}
// idlenessEnforcer is the functionality provided by grpc.ClientConn to enter
// Enforcer is the functionality provided by grpc.ClientConn to enter
// and exit from idle mode.
type idlenessEnforcer interface {
exitIdleMode() error
enterIdleMode() error
type Enforcer interface {
ExitIdleMode() error
EnterIdleMode() error
}
// idlenessManager defines the functionality required to track RPC activity on a
// Manager defines the functionality required to track RPC activity on a
// channel.
type idlenessManager interface {
onCallBegin() error
onCallEnd()
close()
type Manager interface {
OnCallBegin() error
OnCallEnd()
Close()
}
type noopIdlenessManager struct{}
type noopManager struct{}
func (noopIdlenessManager) onCallBegin() error { return nil }
func (noopIdlenessManager) onCallEnd() {}
func (noopIdlenessManager) close() {}
func (noopManager) OnCallBegin() error { return nil }
func (noopManager) OnCallEnd() {}
func (noopManager) Close() {}
// idlenessManagerImpl implements the idlenessManager interface. It uses atomic
// operations to synchronize access to shared state and a mutex to guarantee
// mutual exclusion in a critical section.
type idlenessManagerImpl struct {
// manager implements the Manager interface. It uses atomic operations to
// synchronize access to shared state and a mutex to guarantee mutual exclusion
// in a critical section.
type manager struct {
// State accessed atomically.
lastCallEndTime int64 // Unix timestamp in nanos; time when the most recent RPC completed.
activeCallsCount int32 // Count of active RPCs; -math.MaxInt32 means channel is idle or is trying to get there.
@ -64,14 +68,15 @@ type idlenessManagerImpl struct {
// Can be accessed without atomics or mutex since these are set at creation
// time and read-only after that.
enforcer idlenessEnforcer // Functionality provided by grpc.ClientConn.
timeout int64 // Idle timeout duration nanos stored as an int64.
enforcer Enforcer // Functionality provided by grpc.ClientConn.
timeout int64 // Idle timeout duration nanos stored as an int64.
logger grpclog.LoggerV2
// idleMu is used to guarantee mutual exclusion in two scenarios:
// - Opposing intentions:
// - a: Idle timeout has fired and handleIdleTimeout() is trying to put
// the channel in idle mode because the channel has been inactive.
// - b: At the same time an RPC is made on the channel, and onCallBegin()
// - b: At the same time an RPC is made on the channel, and OnCallBegin()
// is trying to prevent the channel from going idle.
// - Competing intentions:
// - The channel is in idle mode and there are multiple RPCs starting at
@ -83,28 +88,37 @@ type idlenessManagerImpl struct {
timer *time.Timer
}
// newIdlenessManager creates a new idleness manager implementation for the
// ManagerOptions is a collection of options used by
// NewManager.
type ManagerOptions struct {
Enforcer Enforcer
Timeout time.Duration
Logger grpclog.LoggerV2
}
// NewManager creates a new idleness manager implementation for the
// given idle timeout.
func newIdlenessManager(enforcer idlenessEnforcer, idleTimeout time.Duration) idlenessManager {
if idleTimeout == 0 {
return noopIdlenessManager{}
func NewManager(opts ManagerOptions) Manager {
if opts.Timeout == 0 {
return noopManager{}
}
i := &idlenessManagerImpl{
enforcer: enforcer,
timeout: int64(idleTimeout),
m := &manager{
enforcer: opts.Enforcer,
timeout: int64(opts.Timeout),
logger: opts.Logger,
}
i.timer = timeAfterFunc(idleTimeout, i.handleIdleTimeout)
return i
m.timer = timeAfterFunc(opts.Timeout, m.handleIdleTimeout)
return m
}
// resetIdleTimer resets the idle timer to the given duration. This method
// should only be called from the timer callback.
func (i *idlenessManagerImpl) resetIdleTimer(d time.Duration) {
i.idleMu.Lock()
defer i.idleMu.Unlock()
func (m *manager) resetIdleTimer(d time.Duration) {
m.idleMu.Lock()
defer m.idleMu.Unlock()
if i.timer == nil {
if m.timer == nil {
// Only close sets timer to nil. We are done.
return
}
@ -112,47 +126,47 @@ func (i *idlenessManagerImpl) resetIdleTimer(d time.Duration) {
// It is safe to ignore the return value from Reset() because this method is
// only ever called from the timer callback, which means the timer has
// already fired.
i.timer.Reset(d)
m.timer.Reset(d)
}
// handleIdleTimeout is the timer callback that is invoked upon expiry of the
// configured idle timeout. The channel is considered inactive if there are no
// ongoing calls and no RPC activity since the last time the timer fired.
func (i *idlenessManagerImpl) handleIdleTimeout() {
if i.isClosed() {
func (m *manager) handleIdleTimeout() {
if m.isClosed() {
return
}
if atomic.LoadInt32(&i.activeCallsCount) > 0 {
i.resetIdleTimer(time.Duration(i.timeout))
if atomic.LoadInt32(&m.activeCallsCount) > 0 {
m.resetIdleTimer(time.Duration(m.timeout))
return
}
// There has been activity on the channel since we last got here. Reset the
// timer and return.
if atomic.LoadInt32(&i.activeSinceLastTimerCheck) == 1 {
if atomic.LoadInt32(&m.activeSinceLastTimerCheck) == 1 {
// Set the timer to fire after a duration of idle timeout, calculated
// from the time the most recent RPC completed.
atomic.StoreInt32(&i.activeSinceLastTimerCheck, 0)
i.resetIdleTimer(time.Duration(atomic.LoadInt64(&i.lastCallEndTime) + i.timeout - time.Now().UnixNano()))
atomic.StoreInt32(&m.activeSinceLastTimerCheck, 0)
m.resetIdleTimer(time.Duration(atomic.LoadInt64(&m.lastCallEndTime) + m.timeout - time.Now().UnixNano()))
return
}
// This CAS operation is extremely likely to succeed given that there has
// been no activity since the last time we were here. Setting the
// activeCallsCount to -math.MaxInt32 indicates to onCallBegin() that the
// activeCallsCount to -math.MaxInt32 indicates to OnCallBegin() that the
// channel is either in idle mode or is trying to get there.
if !atomic.CompareAndSwapInt32(&i.activeCallsCount, 0, -math.MaxInt32) {
if !atomic.CompareAndSwapInt32(&m.activeCallsCount, 0, -math.MaxInt32) {
// This CAS operation can fail if an RPC started after we checked for
// activity at the top of this method, or one was ongoing from before
// the last time we were here. In both case, reset the timer and return.
i.resetIdleTimer(time.Duration(i.timeout))
m.resetIdleTimer(time.Duration(m.timeout))
return
}
// Now that we've set the active calls count to -math.MaxInt32, it's time to
// actually move to idle mode.
if i.tryEnterIdleMode() {
if m.tryEnterIdleMode() {
// Successfully entered idle mode. No timer needed until we exit idle.
return
}
@ -160,8 +174,8 @@ func (i *idlenessManagerImpl) handleIdleTimeout() {
// Failed to enter idle mode due to a concurrent RPC that kept the channel
// active, or because of an error from the channel. Undo the attempt to
// enter idle, and reset the timer to try again later.
atomic.AddInt32(&i.activeCallsCount, math.MaxInt32)
i.resetIdleTimer(time.Duration(i.timeout))
atomic.AddInt32(&m.activeCallsCount, math.MaxInt32)
m.resetIdleTimer(time.Duration(m.timeout))
}
// tryEnterIdleMode instructs the channel to enter idle mode. But before
@ -171,15 +185,15 @@ func (i *idlenessManagerImpl) handleIdleTimeout() {
// Return value indicates whether or not the channel moved to idle mode.
//
// Holds idleMu which ensures mutual exclusion with exitIdleMode.
func (i *idlenessManagerImpl) tryEnterIdleMode() bool {
i.idleMu.Lock()
defer i.idleMu.Unlock()
func (m *manager) tryEnterIdleMode() bool {
m.idleMu.Lock()
defer m.idleMu.Unlock()
if atomic.LoadInt32(&i.activeCallsCount) != -math.MaxInt32 {
if atomic.LoadInt32(&m.activeCallsCount) != -math.MaxInt32 {
// We raced and lost to a new RPC. Very rare, but stop entering idle.
return false
}
if atomic.LoadInt32(&i.activeSinceLastTimerCheck) == 1 {
if atomic.LoadInt32(&m.activeSinceLastTimerCheck) == 1 {
// An very short RPC could have come in (and also finished) after we
// checked for calls count and activity in handleIdleTimeout(), but
// before the CAS operation. So, we need to check for activity again.
@ -189,99 +203,99 @@ func (i *idlenessManagerImpl) tryEnterIdleMode() bool {
// No new RPCs have come in since we last set the active calls count value
// -math.MaxInt32 in the timer callback. And since we have the lock, it is
// safe to enter idle mode now.
if err := i.enforcer.enterIdleMode(); err != nil {
logger.Errorf("Failed to enter idle mode: %v", err)
if err := m.enforcer.EnterIdleMode(); err != nil {
m.logger.Errorf("Failed to enter idle mode: %v", err)
return false
}
// Successfully entered idle mode.
i.actuallyIdle = true
m.actuallyIdle = true
return true
}
// onCallBegin is invoked at the start of every RPC.
func (i *idlenessManagerImpl) onCallBegin() error {
if i.isClosed() {
// OnCallBegin is invoked at the start of every RPC.
func (m *manager) OnCallBegin() error {
if m.isClosed() {
return nil
}
if atomic.AddInt32(&i.activeCallsCount, 1) > 0 {
if atomic.AddInt32(&m.activeCallsCount, 1) > 0 {
// Channel is not idle now. Set the activity bit and allow the call.
atomic.StoreInt32(&i.activeSinceLastTimerCheck, 1)
atomic.StoreInt32(&m.activeSinceLastTimerCheck, 1)
return nil
}
// Channel is either in idle mode or is in the process of moving to idle
// mode. Attempt to exit idle mode to allow this RPC.
if err := i.exitIdleMode(); err != nil {
if err := m.exitIdleMode(); err != nil {
// Undo the increment to calls count, and return an error causing the
// RPC to fail.
atomic.AddInt32(&i.activeCallsCount, -1)
atomic.AddInt32(&m.activeCallsCount, -1)
return err
}
atomic.StoreInt32(&i.activeSinceLastTimerCheck, 1)
atomic.StoreInt32(&m.activeSinceLastTimerCheck, 1)
return nil
}
// exitIdleMode instructs the channel to exit idle mode.
//
// Holds idleMu which ensures mutual exclusion with tryEnterIdleMode.
func (i *idlenessManagerImpl) exitIdleMode() error {
i.idleMu.Lock()
defer i.idleMu.Unlock()
func (m *manager) exitIdleMode() error {
m.idleMu.Lock()
defer m.idleMu.Unlock()
if !i.actuallyIdle {
if !m.actuallyIdle {
// This can happen in two scenarios:
// - handleIdleTimeout() set the calls count to -math.MaxInt32 and called
// tryEnterIdleMode(). But before the latter could grab the lock, an RPC
// came in and onCallBegin() noticed that the calls count is negative.
// came in and OnCallBegin() noticed that the calls count is negative.
// - Channel is in idle mode, and multiple new RPCs come in at the same
// time, all of them notice a negative calls count in onCallBegin and get
// time, all of them notice a negative calls count in OnCallBegin and get
// here. The first one to get the lock would got the channel to exit idle.
//
// Either way, nothing to do here.
return nil
}
if err := i.enforcer.exitIdleMode(); err != nil {
if err := m.enforcer.ExitIdleMode(); err != nil {
return fmt.Errorf("channel failed to exit idle mode: %v", err)
}
// Undo the idle entry process. This also respects any new RPC attempts.
atomic.AddInt32(&i.activeCallsCount, math.MaxInt32)
i.actuallyIdle = false
atomic.AddInt32(&m.activeCallsCount, math.MaxInt32)
m.actuallyIdle = false
// Start a new timer to fire after the configured idle timeout.
i.timer = timeAfterFunc(time.Duration(i.timeout), i.handleIdleTimeout)
m.timer = timeAfterFunc(time.Duration(m.timeout), m.handleIdleTimeout)
return nil
}
// onCallEnd is invoked at the end of every RPC.
func (i *idlenessManagerImpl) onCallEnd() {
if i.isClosed() {
// OnCallEnd is invoked at the end of every RPC.
func (m *manager) OnCallEnd() {
if m.isClosed() {
return
}
// Record the time at which the most recent call finished.
atomic.StoreInt64(&i.lastCallEndTime, time.Now().UnixNano())
atomic.StoreInt64(&m.lastCallEndTime, time.Now().UnixNano())
// Decrement the active calls count. This count can temporarily go negative
// when the timer callback is in the process of moving the channel to idle
// mode, but one or more RPCs come in and complete before the timer callback
// can get done with the process of moving to idle mode.
atomic.AddInt32(&i.activeCallsCount, -1)
atomic.AddInt32(&m.activeCallsCount, -1)
}
func (i *idlenessManagerImpl) isClosed() bool {
return atomic.LoadInt32(&i.closed) == 1
func (m *manager) isClosed() bool {
return atomic.LoadInt32(&m.closed) == 1
}
func (i *idlenessManagerImpl) close() {
atomic.StoreInt32(&i.closed, 1)
func (m *manager) Close() {
atomic.StoreInt32(&m.closed, 1)
i.idleMu.Lock()
i.timer.Stop()
i.timer = nil
i.idleMu.Unlock()
m.idleMu.Lock()
m.timer.Stop()
m.timer = nil
m.idleMu.Unlock()
}

View File

@ -30,7 +30,7 @@ import (
var (
// WithHealthCheckFunc is set by dialoptions.go
WithHealthCheckFunc interface{} // func (HealthChecker) DialOption
WithHealthCheckFunc any // func (HealthChecker) DialOption
// HealthCheckFunc is used to provide client-side LB channel health checking
HealthCheckFunc HealthChecker
// BalancerUnregister is exported by package balancer to unregister a balancer.
@ -38,8 +38,12 @@ var (
// KeepaliveMinPingTime is the minimum ping interval. This must be 10s by
// default, but tests may wish to set it lower for convenience.
KeepaliveMinPingTime = 10 * time.Second
// KeepaliveMinServerPingTime is the minimum ping interval for servers.
// This must be 1s by default, but tests may wish to set it lower for
// convenience.
KeepaliveMinServerPingTime = time.Second
// ParseServiceConfig parses a JSON representation of the service config.
ParseServiceConfig interface{} // func(string) *serviceconfig.ParseResult
ParseServiceConfig any // func(string) *serviceconfig.ParseResult
// EqualServiceConfigForTesting is for testing service config generation and
// parsing. Both a and b should be returned by ParseServiceConfig.
// This function compares the config without rawJSON stripped, in case the
@ -49,33 +53,33 @@ var (
// given name. This is set by package certprovider for use from xDS
// bootstrap code while parsing certificate provider configs in the
// bootstrap file.
GetCertificateProviderBuilder interface{} // func(string) certprovider.Builder
GetCertificateProviderBuilder any // func(string) certprovider.Builder
// GetXDSHandshakeInfoForTesting returns a pointer to the xds.HandshakeInfo
// stored in the passed in attributes. This is set by
// credentials/xds/xds.go.
GetXDSHandshakeInfoForTesting interface{} // func (*attributes.Attributes) *xds.HandshakeInfo
GetXDSHandshakeInfoForTesting any // func (*attributes.Attributes) *xds.HandshakeInfo
// GetServerCredentials returns the transport credentials configured on a
// gRPC server. An xDS-enabled server needs to know what type of credentials
// is configured on the underlying gRPC server. This is set by server.go.
GetServerCredentials interface{} // func (*grpc.Server) credentials.TransportCredentials
GetServerCredentials any // func (*grpc.Server) credentials.TransportCredentials
// CanonicalString returns the canonical string of the code defined here:
// https://github.com/grpc/grpc/blob/master/doc/statuscodes.md.
//
// This is used in the 1.0 release of gcp/observability, and thus must not be
// deleted or changed.
CanonicalString interface{} // func (codes.Code) string
CanonicalString any // func (codes.Code) string
// DrainServerTransports initiates a graceful close of existing connections
// on a gRPC server accepted on the provided listener address. An
// xDS-enabled server invokes this method on a grpc.Server when a particular
// listener moves to "not-serving" mode.
DrainServerTransports interface{} // func(*grpc.Server, string)
DrainServerTransports any // func(*grpc.Server, string)
// AddGlobalServerOptions adds an array of ServerOption that will be
// effective globally for newly created servers. The priority will be: 1.
// user-provided; 2. this method; 3. default values.
//
// This is used in the 1.0 release of gcp/observability, and thus must not be
// deleted or changed.
AddGlobalServerOptions interface{} // func(opt ...ServerOption)
AddGlobalServerOptions any // func(opt ...ServerOption)
// ClearGlobalServerOptions clears the array of extra ServerOption. This
// method is useful in testing and benchmarking.
//
@ -88,14 +92,14 @@ var (
//
// This is used in the 1.0 release of gcp/observability, and thus must not be
// deleted or changed.
AddGlobalDialOptions interface{} // func(opt ...DialOption)
AddGlobalDialOptions any // func(opt ...DialOption)
// DisableGlobalDialOptions returns a DialOption that prevents the
// ClientConn from applying the global DialOptions (set via
// AddGlobalDialOptions).
//
// This is used in the 1.0 release of gcp/observability, and thus must not be
// deleted or changed.
DisableGlobalDialOptions interface{} // func() grpc.DialOption
DisableGlobalDialOptions any // func() grpc.DialOption
// ClearGlobalDialOptions clears the array of extra DialOption. This
// method is useful in testing and benchmarking.
//
@ -104,23 +108,26 @@ var (
ClearGlobalDialOptions func()
// JoinDialOptions combines the dial options passed as arguments into a
// single dial option.
JoinDialOptions interface{} // func(...grpc.DialOption) grpc.DialOption
JoinDialOptions any // func(...grpc.DialOption) grpc.DialOption
// JoinServerOptions combines the server options passed as arguments into a
// single server option.
JoinServerOptions interface{} // func(...grpc.ServerOption) grpc.ServerOption
JoinServerOptions any // func(...grpc.ServerOption) grpc.ServerOption
// WithBinaryLogger returns a DialOption that specifies the binary logger
// for a ClientConn.
//
// This is used in the 1.0 release of gcp/observability, and thus must not be
// deleted or changed.
WithBinaryLogger interface{} // func(binarylog.Logger) grpc.DialOption
WithBinaryLogger any // func(binarylog.Logger) grpc.DialOption
// BinaryLogger returns a ServerOption that can set the binary logger for a
// server.
//
// This is used in the 1.0 release of gcp/observability, and thus must not be
// deleted or changed.
BinaryLogger interface{} // func(binarylog.Logger) grpc.ServerOption
BinaryLogger any // func(binarylog.Logger) grpc.ServerOption
// SubscribeToConnectivityStateChanges adds a grpcsync.Subscriber to a provided grpc.ClientConn
SubscribeToConnectivityStateChanges any // func(*grpc.ClientConn, grpcsync.Subscriber)
// NewXDSResolverWithConfigForTesting creates a new xds resolver builder using
// the provided xds bootstrap config instead of the global configuration from
@ -131,7 +138,7 @@ var (
//
// This function should ONLY be used for testing and may not work with some
// other features, including the CSDS service.
NewXDSResolverWithConfigForTesting interface{} // func([]byte) (resolver.Builder, error)
NewXDSResolverWithConfigForTesting any // func([]byte) (resolver.Builder, error)
// RegisterRLSClusterSpecifierPluginForTesting registers the RLS Cluster
// Specifier Plugin for testing purposes, regardless of the XDSRLS environment
@ -163,7 +170,11 @@ var (
UnregisterRBACHTTPFilterForTesting func()
// ORCAAllowAnyMinReportingInterval is for examples/orca use ONLY.
ORCAAllowAnyMinReportingInterval interface{} // func(so *orca.ServiceOptions)
ORCAAllowAnyMinReportingInterval any // func(so *orca.ServiceOptions)
// GRPCResolverSchemeExtraMetadata determines when gRPC will add extra
// metadata to RPCs.
GRPCResolverSchemeExtraMetadata string = "xds"
)
// HealthChecker defines the signature of the client-side LB channel health checking function.
@ -174,7 +185,7 @@ var (
//
// The health checking protocol is defined at:
// https://github.com/grpc/grpc/blob/master/doc/health-checking.md
type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State, error), serviceName string) error
type HealthChecker func(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), serviceName string) error
const (
// CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode.

View File

@ -35,7 +35,7 @@ const mdKey = mdKeyType("grpc.internal.address.metadata")
type mdValue metadata.MD
func (m mdValue) Equal(o interface{}) bool {
func (m mdValue) Equal(o any) bool {
om, ok := o.(mdValue)
if !ok {
return false

View File

@ -35,7 +35,7 @@ const jsonIndent = " "
// ToJSON marshals the input into a json string.
//
// If marshal fails, it falls back to fmt.Sprintf("%+v").
func ToJSON(e interface{}) string {
func ToJSON(e any) string {
switch ee := e.(type) {
case protov1.Message:
mm := jsonpb.Marshaler{Indent: jsonIndent}

Some files were not shown because too many files have changed in this diff Show More