upgrade to latest dependencies (#1773)

bumping golang.org/x/time 2ce7c29...f3bd1da:
  > f3bd1da rate: add TokenAt and Tokens methods to Limiter.
bumping knative.dev/serving 484e848...587f587:
  > 587f587 add missing probes (# 13563)
  > 5e3b4af Adds e2e test for activation-scale (# 13197)
  > b285cb4 Update net-certmanager nightly (# 13762)
  > aab7abe Update net-contour nightly (# 13761)
  > 0cf0da8 Update net-gateway-api nightly (# 13756)
  > efc0bb0 Update net-kourier nightly (# 13760)
  > 6bc4bb2 upgrade to latest dependencies (# 13759)
  > 59d4c52 Update net-kourier nightly (# 13755)
  > 5a7a72e Update net-contour nightly (# 13758)
  > cc58860 Update net-certmanager nightly (# 13757)
  > 879c723 upgrade to latest dependencies (# 13742)
  > 23212b3 Update net-kourier nightly (# 13733)
  > 12b0690 Update net-contour nightly (# 13734)
  > 628b3e6 Update net-gateway-api nightly (# 13732)
  > cd3fe03 Update net-certmanager nightly (# 13735)
  > 28b8fba upgrade to latest dependencies (# 13730)
  > 708374e TestAutoscaleSustaining scales to 8 instead of 10 (# 13679)
  > 168ea39 Update net-contour nightly (# 13729)
  > f2ee629 Update net-certmanager nightly (# 13728)
  > bd75129 upgrade to latest dependencies (# 13726)
  > f4792e4 check for renewing status (# 13666)
  > 51ed9ad Update net-contour nightly (# 13724)
  > 1b99dce Update net-gateway-api nightly (# 13719)
  > fe52019 Update net-certmanager nightly (# 13718)
  > 506f8c6 Update net-contour nightly (# 13717)
  > 1f5f822 Update net-kourier nightly (# 13698)
  > 7aeb55d Update net-gateway-api nightly (# 13716)
  > aa4149f Update net-certmanager nightly (# 13715)
  > 04da752 upgrade to latest dependencies (# 13713)
  > dfd39ca Update support rotation (# 13714)
  > 8c282ba Update net-certmanager nightly (# 13697)
  > 7ed2869 Update support rotation contacts (# 13703)
  > 0ec0ecf Update net-gateway-api nightly (# 13699)
  > 060a49a Update net-contour nightly (# 13705)
  > 53e91c9 Fix G112: Potential Slowloris Attacks lint errs (# 13702)
  > a99a936 upgrade to latest dependencies (# 13709)
  > 08812b3 Update net-istio nightly (# 13706)
  > 368994b Update community files (# 13707)
  > d68797e Bumping 'github.com/google/go-containerregistry' dependency (# 13701)
  > 9b9a951 upgrade to latest dependencies (# 13689)
  > e2add5d fix curl invocation (# 13683)
  > 6334ffe Update net-contour nightly (# 13676)
  > d7959af Update net-certmanager nightly (# 13686)
  > 4db2ada Update net-istio nightly (# 13687)
  > 7f2264d Update community files (# 13685)
  > baa6292 Update net-gateway-api nightly (# 13672)
  > 83d2dd7 Update net-kourier nightly (# 13673)
  > 2fc2a12 fix route reconciler test flakes (# 13684)
  > 729eb6d Update net-istio nightly (# 13674)
  > 30a2b0c Add k8s 1.26 for kind (# 13682)
  > 3764762 use GITHUB_TOKEN when querying net-istio releases (# 13681)
  > 485f2a9 upgrade to latest dependencies (# 13680)
  > 0639c5f Update community files (# 13678)
  > b4d7a28 clean up OWNER files (# 13668)
  > 68e128b Update net-contour nightly (# 13667)
  > f6ab591 Adds slack links to CNCF Knative slack channel (# 13655)
  > 53b3c92 Add serving-internal docs about Knative encryption support (# 13662)
  > 3b8d6cb Update net-contour nightly (# 13656)
  > b9b4c57 Update net-gateway-api nightly (# 13654)
  > 63832e2 Update net-contour nightly (# 13650)
  > 908edaf Update net-istio nightly (# 13649)
  > 06add5f Update net-kourier nightly (# 13648)
bumping knative.dev/networking db2bcbe...c692e9e:
  > c692e9e upgrade to latest dependencies (# 776)
  > 475e232 upgrade to latest dependencies (# 775)
  > 5a5f810 upgrade to latest dependencies (# 767)
  > 2c2695b upgrade to latest dependencies (# 766)
  > 1bd50a5 Update community files (# 764)
  > 6a15e7d upgrade to latest dependencies (# 765)
  > 9373ba6 upgrade to latest dependencies (# 762)
  > 195809a Update community files (# 761)
  > c3510af upgrade to latest dependencies (# 760)
  > 2473e65 Update community files (# 759)
  > f6585dc update OWNERS file (# 758)
  > 89f3990 Assert all the expected DNSNames are part of the HTTP01 challenge (# 756)
  > 4ccbe60 adjust domain validation in kcert (# 754)
bumping github.com/google/go-containerregistry 31786c6...1e09daa:
  > 1e09daa clarify crane download readme (# 1533)
  > 8e08d51 Revert "Hack around DockerHub plugin scope handling" (# 1531)
  > 5ad0a76 crane: support --full-ref for crane ls (# 1525)
  > b063f6a Hack around DockerHub plugin scope handling (# 1527)
  > e797859 crane: add digest --full (# 1524)
  > 3986cf4 test: use `T.TempDir` to create temporary test directory (# 1522)
  > 9db616f FIX mutate.Time not respecting history (# 1520)
  > 8048663 Race http fallback ping (# 1521)
  > e64ff3a Bump actions/stale from 6 to 7 (# 1519)
  > 9bd8237 Bump goreleaser/goreleaser-action from 3.2.0 to 4.1.0 (# 1511)
  > ffc14a0 Treat empty registry config as anonymous (# 1512)
  > 23d895d Fix missing doc comment (# 1509)
  > 6ba20c8 Fix various lints (# 1507)
  > 4270e04 Update Arch Linux install instructions (# 1508)
  > 37bf5df Revert "docs: pull latest instead of debug (# 1497)" (# 1504)
  > 47f0933 Use the default retry predicate in transport (# 1502)
  > c7270c2 Make unit tests substantially faster (# 1498)
  > a35805d Make credential warning slightly more accurate (# 1499)
  > 37b993a docs: pull latest instead of debug (# 1497)
  > cd77615 Add support for zstd compression (# 1487)
  > c412593 Fix calculating tarball size when duplicated layers exist (# 1495)
  > bdc946f add source archive checksum into the checksums.txt (# 1492)
  > 781782a Bump golangci/golangci-lint-action from 3.3.0 to 3.3.1 (# 1491)
  > 9e939fb Features: Allow eliding `serviceaccount` lookups. (# 1490)
  > 3e4f490 Bump slsa-framework/slsa-github-generator to 1.2.2 (# 1489)
  > 76ae819 Fix context.DeadlineExceeded comparison (# 1488)
  > 353a117 crane: add catalog argument use annotation (# 1473)
  > 1711cef Fix missing body.Close() in bearer auth (# 1482)
  > 426de7d Bump golangci/golangci-lint-action from 3.2.0 to 3.3.0 (# 1475)
  > 6442b02 Bump goreleaser/goreleaser-action from 3.1.0 to 3.2.0 (# 1476)
  > a0cca8a k8schain: Log and proceed if secret or SA are not found (# 1472)
  > 02f47e1 bump version of slsa generator (# 1468)
  > 3413eb6 registry: implement pagination (# 1430)
  > d3ed408 registry: implement blob deletion (# 1432)
  > 969699e Bump deps using ./hack/bump-deps.sh (# 1467)
  > 9b4fdd5 Bump actions/setup-go from 2 to 3 (# 1463)
  > a0f6687 Make ErrBadName checkable via errors.Is() (# 1462)
  > 7268da0 Bump actions/stale from 5 to 6 (# 1452)
  > 8eae069 Bump codecov/codecov-action from 3.1.0 to 3.1.1 (# 1453)
  > 9a5c14a fix crane's root.go after DefaultTransport change (# 1450)
  > e3b94c7 allow remote.DefaultTransport to be overridden by an http.RoundTripper (# 1449)
  > f981b4c deps: update goreleaser-action for bug (# 1444)
  > 87b3a79 feat: Add krane to release archive (# 1443)
  > e2d575c update crane installation instructions and release verification (# 1440)
  > 2859a0d feat: generate slsa provenance on github release artifacts (# 1438)
  > 771a9b4 e2e: pull and export stdin and stdout (# 1436)
  > 2b54510 fix: consider base image media type when appending layers (# 1437)
  > 7196cf3 Bump aws-actions/configure-aws-credentials from 1.6.1 to 1.7.0 (# 1424)
  > c1f9836 Bump opencontainers/image-spec (# 1423)
  > 24a1c33 Ignore docker config if it's a directory (# 1420)
  > 49cdb8b Correct usage of authn.NewKeychainFromHelper in docs (# 1419)
  > 19e3eff Retry ECONNRESET errors (# 1415)
  > 3ba4c51 Fix tar PAX format handling (# 1414)
  > 5749ee6 Support the platform specific authentication of krane in "auth get" command (# 1413)
bumping golang.org/x/crypto 35f4265...642fcc3:
  > 642fcc3 go.mod: update golang.org/x dependencies
  > 56aed06 all: use automatic RFC linking
  > 9be5aaa all: fix a few function names on comments
  > d6f0a8c ssh: add ServerConfig.NoClientAuthCallback
  > 4161e89 all: replace bytes.Compare with bytes.Equal
  > eccd636 acme/autocert: remove TestRenewFromCache skips
  > 4ba4fb4 acme/autocert: fix renewal timer issue
bumping knative.dev/eventing 4d6e1fc...1ff36e1:
  > 1ff36e1 Support testing ContainerSource with istio (# 6790)
  > a8128e6 [main] Upgrade to latest dependencies (# 6774)
  > 397387e SC settings for ApiServerSource's Receive Adapter's container/deployment  (# 6788)
  > b474770 No reason for API ServerSource adapter to not inject istio sidecar (# 6789)
  > 7d7df2d Removes the deprecated DeadLetterChannel in ChannelableStatus (# 6722)
  > c5847a9 Port broker many triggers testcase to rekt (# 6761)
  > 52574ce Fixes issue where a CE response is truncated (# 6758)
  > 44b0b8a Remove deprecated usage of ZipkinTracingEnabled (# 6780)
  > 07e6d0c [main] Upgrade to latest dependencies (# 6770)
  > 70d2cb9 Pass EnvConfig to Broker conformance tests (# 6769)
  > f6afad3 Exclusive access to tracing flag for upgrade prober (# 6767)
  > 843b6f2 Allow event display to log requests (# 6764)
  > 23dc742 [main] Upgrade to latest dependencies (# 6765)
  > 82d85f9 [main] Upgrade to latest dependencies (# 6762)
  > 5f24569 [main] Upgrade to latest dependencies (# 6744)
  > dac6b8b [main] Update community files (# 6756)
  > 2b517b6 Deprecate `test/rekt/resources/svc` (# 6742)
  > bbbd425 Deprecate `test/rekt/resources/deployment` (# 6748)
  > f022034 Update dependencies (# 6753)
  > fdffabe Deprecate `test/rekt/resources/pod` package (# 6747)
  > dfd095f Remove event flaker since eventshub offers an equivalent feature (# 6750)
  > 0ab29eb  Remove unused event-library (# 6751)
  > 82324b9 [main] Update community files (# 6743)
  > 452cd49 Adds apiVersion as an extension to api source events (# 6696)
  > b96d2fb Migrate from `knative.dev/reconciler-test/resources/svc` to `knative.dev/reconciler-test/pkg/resources/service` (# 6741)
  > f9ceb4f [main] Upgrade to latest dependencies (# 6715)
  > 37fa6e0 Extract scheduler config in a dedicate struct instead of many parameters (# 6736)
  > 4911986 Add function to check if a broker resource is `NotReady` (# 6737)
  > 2f98e55 Improve scheduler logging for state and pending vpods (# 6729)
  > 1092472 Scheduler doesn't reschedule vpods that are scheduled on unscehdulable pods (# 6726)
  > 40517be Added Broker class in `kubectl get -o wide` (# 6723)
  > e609459 [main] Update community files (# 6727)
  > 47a793b Revert "Change subscription patch logic to ensure resource version (# 6670) (# 6724)
  > 8b7551c Removing one more deprecated, unused func (# 6718)
  > 228eda3 Removing deprecated and unused func (# 6716)
  > ffc8fe0 Default a subscriptions subscriber and reply in webhook (# 6701)
  > 8d62a06  Adds link to the Knative CNCF Slack Channel  (# 6711)
  > 91a1991 Install source at requirement phase in conformance tests (# 6712)
  > 983189d Set scheduler logging to debug (# 6705)
  > c3364e4 Remove UNUSED and deprecated test helpers (# 6710)
  > 9b8c4d6 Port BrokerWorkFlowWithTranformation test to rekt and code cleanup (# 6709)
  > 45e7a24 Do not overwrite existing imagePullSecrets in CopySecret (# 6707)
  > efba019 [main] Format Go code (# 6702)
bumping golang.org/x/sync f12130a...8fcdb60:
  > 8fcdb60 singleflight: avoid race between multiple Do calls
  > 7f9b162 singleflight: remove forgotten field
bumping golang.org/x/mod 86c51ed...b3066c3:
  > b3066c3 go.mod: update golang.org/x dependencies
  > e3c1277 go.mod: update to tagged x/tools version
  > aac77cd all: fix a few function names on comments
  > 2666ed6 go.mod: ignore cyclic dependency for tagging
  > 2adab6b zip: expand logging and use t.TempDir and t.Cleanup in test helpers
  > 02c9913 sumdb: remove redundant type conversion
  > 8f535f7 sumdb/note: fix some typos
  > ed83ed6 modfile: remove duplicate words from comments
  > f994a2a zip: set PWD consistently for commands in subdirectories
  > 046e8b3 modfile: improve error message for replace with '@' in path
bumping knative.dev/pkg 247510c...b77a78c:
  > b77a78c upgrade to latest dependencies (# 2698)
  > 310274d bump golang version in downstream tests (# 2697)
  > fd00e03 Add certs secret name read from env (# 2685)
  > 645afb2 bump golang.org/x/net to v0.7.0 (# 2695)
  > 0591b75 bump net and text packages (# 2692)
  > 2fdd6bf Allow reconcilers to listen to leader promotion events  (# 2688)
  > f65eba5 Make ZipkinTracingEnabled flag thread safe (# 2690)
  > f306c13 upgrade to latest dependencies (# 2684)
  > 2f55fe0 Fix potential slowloris attack (# 2682)
  > 8d1efe1 Update community files (# 2683)
  > 4a80605 Add health checks (# 2671)
  > 33e6b88 fix: `reconcilerImpl.updateStatus` calculates state difference in debug mode only (# 2678)
  > c889c5b upgrade to latest dependencies (# 2679)
  > fb44e94 Update community files (# 2676)
  > decc1cc Update community files (# 2674)
  > 8532ae0 update OWNERS files to use teams with active members (# 2672)
  > 408ad07 Make SetDefaults of Destination duck type nil safer (# 2670)
bumping k8s.io/kube-openapi 67bda5d...172d655:
  > 172d655 Merge pull request # 319 from alexzielenski/json_next
  > 829ce0c fix json unmarshal bug when extension used with Responses (# 320)
  > 8d49439 add JSONRoundTripTestCase and add a Test*RoundTrip for every spec.* type which needs it
  > b7a9e6f Merge pull request # 318 from dims/add-go.mod-for-integration-tests
  > 3b3bb66 add jsonv2 roundtrip fuzz tests and benchmarks
  > a70c9af Merge pull request # 307 from apelisse/test-required-job
  > b856c89 Add a go.mod for integration test
  > 395774f add jsonv2 implementation
  > c6e407d Add required job that depends on matrix jobs
  > c601e50 add go-json-experiment/json to pkg/internal/third_party
  > 5ca72f5 gofmt changes

Signed-off-by: Knative Automation <automation@knative.team>
This commit is contained in:
knative-automation 2023-03-07 04:27:19 -05:00 committed by GitHub
parent e82e7497da
commit 3bdc2329a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
69 changed files with 10712 additions and 189 deletions

26
go.mod
View File

@ -11,7 +11,7 @@ require (
github.com/spf13/cobra v1.6.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.13.0
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
golang.org/x/mod v0.6.0
golang.org/x/term v0.5.0
gotest.tools/v3 v3.3.0
k8s.io/api v0.25.4
@ -20,11 +20,11 @@ require (
k8s.io/cli-runtime v0.25.2
k8s.io/client-go v0.25.4
k8s.io/code-generator v0.25.4
knative.dev/eventing v0.36.0
knative.dev/hack v0.0.0-20230113013652-c7cfcb062de9
knative.dev/networking v0.0.0-20230123233838-db2bcbea2560
knative.dev/pkg v0.0.0-20230117181655-247510c00e9d
knative.dev/serving v0.36.0
knative.dev/eventing v0.36.1-0.20230306130433-1ff36e1b656d
knative.dev/hack v0.0.0-20230228173453-3de51aff69a3
knative.dev/networking v0.0.0-20230301131055-c692e9e6afe1
knative.dev/pkg v0.0.0-20230306194819-b77a78c6c0ad
knative.dev/serving v0.36.1-0.20230306232326-587f58791d1b
sigs.k8s.io/yaml v1.3.0
)
@ -47,7 +47,7 @@ require (
github.com/cloudevents/sdk-go/v2 v2.13.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.8.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
@ -64,7 +64,7 @@ require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-containerregistry v0.11.0 // indirect
github.com/google/go-containerregistry v0.13.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
@ -108,13 +108,13 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
golang.org/x/oauth2 v0.1.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 // indirect
golang.org/x/time v0.1.0 // indirect
golang.org/x/tools v0.1.12 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/api v0.96.0 // indirect
@ -128,7 +128,7 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/gengo v0.0.0-20221011193443-fad74ee6edd9 // indirect
k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 // indirect
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/kustomize/api v0.12.1 // indirect

52
go.sum
View File

@ -136,8 +136,8 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw=
github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@ -256,8 +256,8 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.11.0 h1:Xt8x1adcREjFcmDoDK8OdOsjxu90PHkGuwNP8GiHMLM=
github.com/google/go-containerregistry v0.11.0/go.mod h1:BBaYtsHPHA42uEgAvd/NejvAfPSlz281sJWqupjSxfk=
github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k=
github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
@ -544,8 +544,8 @@ golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY=
golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -581,8 +581,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -660,8 +660,8 @@ golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA=
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y=
golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -674,8 +674,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A=
golang.org/x/sync v0.0.0-20220907140024-f12130a52804/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -774,8 +774,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 h1:yuLAip3bfURHClMG9VBdzPrQvCWjWiWUTBGV+/fCbUs=
golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA=
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -1083,20 +1083,20 @@ k8s.io/gengo v0.0.0-20221011193443-fad74ee6edd9/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 h1:hWRbsoRWt44OEBnYUd4ceLy4ofBoh+p9vauWp/I5Gdg=
k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 h1:GfD9OzL11kvZN5iArC6oTS7RTj7oJOIfnislxYlqTj8=
k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
knative.dev/eventing v0.36.0 h1:a7kamc2S+LcpNMDX3llnwZm+DqMcYSXgKIgJXdaQQSY=
knative.dev/eventing v0.36.0/go.mod h1:Qka5Z6+LeMoHGL1QAznVdmq5LAu21b4F3rgxc2AMgRg=
knative.dev/hack v0.0.0-20230113013652-c7cfcb062de9 h1:CDa7s9KspEZqPhk7cN68ZypRLuAvSgr+knoOaXSsrHk=
knative.dev/hack v0.0.0-20230113013652-c7cfcb062de9/go.mod h1:yk2OjGDsbEnQjfxdm0/HJKS2WqTLEFg/N6nUs6Rqx3Q=
knative.dev/networking v0.0.0-20230123233838-db2bcbea2560 h1:iprdS5tKTXtgV9dGryuwJJJTTdl5LusCHOelKdezR3I=
knative.dev/networking v0.0.0-20230123233838-db2bcbea2560/go.mod h1:rn1yRurhkxmSFkpqs/YdG7b9DiYj0VlmLFzBdOQjpOo=
knative.dev/pkg v0.0.0-20230117181655-247510c00e9d h1:pjKDcvHoMib8nRp56eISRmMj/pFMzJljnzvMvGCIReI=
knative.dev/pkg v0.0.0-20230117181655-247510c00e9d/go.mod h1:VO/fcEsq43seuONRQxZyftWHjpMabYzRHDtpSEQ/eoQ=
knative.dev/serving v0.36.0 h1:RSYDjxhzOx5rnlW9tNPcBPyJyNuOcZuYEMdKDR1r04k=
knative.dev/serving v0.36.0/go.mod h1:ueqMvTqzZE0GFfPqSsc+ZjX20Z8XxCuX86+S+TI7B3A=
knative.dev/eventing v0.36.1-0.20230306130433-1ff36e1b656d h1:mYftXlassunWeX4gvnMZMheqjMnixe4x89guq5atJ48=
knative.dev/eventing v0.36.1-0.20230306130433-1ff36e1b656d/go.mod h1:wvjeFSAkuYET+tHkS3EdKcPsLxgvHK6qEkdzxR4ffkw=
knative.dev/hack v0.0.0-20230228173453-3de51aff69a3 h1:Jt3n+AZsZHZaFhk/A1NnboAvgjV+hvvyeyyuIX/hUx0=
knative.dev/hack v0.0.0-20230228173453-3de51aff69a3/go.mod h1:yk2OjGDsbEnQjfxdm0/HJKS2WqTLEFg/N6nUs6Rqx3Q=
knative.dev/networking v0.0.0-20230301131055-c692e9e6afe1 h1:ejM3dZpJSGWPXD4E/ZrL3baOU9fBD3neJkIGKs+DE8M=
knative.dev/networking v0.0.0-20230301131055-c692e9e6afe1/go.mod h1:v3IQxqBhOuTMDFZhsvBFGiuF5vjlOGoSQM2Q6B7HSpU=
knative.dev/pkg v0.0.0-20230306194819-b77a78c6c0ad h1:cYCDdgSMOKiCGm6v1vvR2v4l/naGorbwoJKE/e39BJI=
knative.dev/pkg v0.0.0-20230306194819-b77a78c6c0ad/go.mod h1:S+KfTInuwEkZSTwvWqrWZV/TEw6ps51GUGaSC1Fnbe0=
knative.dev/serving v0.36.1-0.20230306232326-587f58791d1b h1:tII98txShIZRsBIJeYbKi4QT6Pncd+1VCluZ2QCpMR4=
knative.dev/serving v0.36.1-0.20230306232326-587f58791d1b/go.mod h1:fBbRIhI0BXjoMJECg0ac+ifIdyKKfu9ZGDsbB5isK4w=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

View File

@ -0,0 +1,27 @@
Copyright (c) 2020 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,5 +1,9 @@
# Change history of go-restful
## [v3.9.0] - 20221-07-21
- add support for http.Handler implementations to work as FilterFunction, issue #504 (thanks to https://github.com/ggicci)
## [v3.8.0] - 20221-06-06
- use exact matching of allowed domain entries, issue #489 (#493)

View File

@ -84,6 +84,7 @@ func (u UserResource) findUser(request *restful.Request, response *restful.Respo
- Route errors produce HTTP 404/405/406/415 errors, customizable using ServiceErrorHandler(...)
- Configurable (trace) logging
- Customizable gzip/deflate readers and writers using CompressorProvider registration
- Inject your own http.Handler using the `HttpMiddlewareHandlerToFilter` function
## How to customize
There are several hooks to customize the behavior of the go-restful package.
@ -94,7 +95,7 @@ There are several hooks to customize the behavior of the go-restful package.
- Trace logging
- Compression
- Encoders for other serializers
- Use [jsoniter](https://github.com/json-iterator/go) by build this package using a tag, e.g. `go build -tags=jsoniter .`
- Use [jsoniter](https://github.com/json-iterator/go) by building this package using a build tag, e.g. `go build -tags=jsoniter .`
## Resources

View File

@ -0,0 +1,21 @@
package restful
import (
"net/http"
)
// HttpMiddlewareHandler is a function that takes a http.Handler and returns a http.Handler
type HttpMiddlewareHandler func(http.Handler) http.Handler
// HttpMiddlewareHandlerToFilter converts a HttpMiddlewareHandler to a FilterFunction.
func HttpMiddlewareHandlerToFilter(middleware HttpMiddlewareHandler) FilterFunction {
return func(req *Request, resp *Response, chain *FilterChain) {
next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
req.Request = r
resp.ResponseWriter = rw
chain.ProcessFilter(req, resp)
})
middleware(next).ServeHTTP(resp.ResponseWriter, req.Request)
}
}

View File

@ -22,6 +22,9 @@ const (
// FormParameterKind = indicator of Request parameter type "form"
FormParameterKind
// MultiPartFormParameterKind = indicator of Request parameter type "multipart/form-data"
MultiPartFormParameterKind
// CollectionFormatCSV comma separated values `foo,bar`
CollectionFormatCSV = CollectionFormat("csv")
@ -108,6 +111,11 @@ func (p *Parameter) beForm() *Parameter {
return p
}
func (p *Parameter) beMultiPartForm() *Parameter {
p.data.Kind = MultiPartFormParameterKind
return p
}
// Required sets the required field and returns the receiver
func (p *Parameter) Required(required bool) *Parameter {
p.data.Required = required

View File

@ -165,6 +165,18 @@ func FormParameter(name, description string) *Parameter {
return p
}
// MultiPartFormParameter creates a new Parameter of kind Form (using multipart/form-data) for documentation purposes.
// It is initialized as required with string as its DataType.
func (w *WebService) MultiPartFormParameter(name, description string) *Parameter {
return MultiPartFormParameter(name, description)
}
func MultiPartFormParameter(name, description string) *Parameter {
p := &Parameter{&ParameterData{Name: name, Description: description, Required: false, DataType: "string"}}
p.beMultiPartForm()
return p
}
// Route creates a new Route using the RouteBuilder and add to the ordered list of Routes.
func (w *WebService) Route(builder *RouteBuilder) *WebService {
w.routesLock.Lock()

View File

@ -28,8 +28,14 @@ func (e *ErrBadName) Error() string {
return e.info
}
// Is reports whether target is an error of type ErrBadName
func (e *ErrBadName) Is(target error) bool {
var berr *ErrBadName
return errors.As(target, &berr)
}
// newErrBadName returns a ErrBadName which returns the given formatted string from Error().
func newErrBadName(fmtStr string, args ...interface{}) *ErrBadName {
func newErrBadName(fmtStr string, args ...any) *ErrBadName {
return &ErrBadName{fmt.Sprintf(fmtStr, args...)}
}

View File

@ -56,16 +56,16 @@ type stringConst string
// To discourage its use in scenarios where the value is not known at code
// authoring time, it must be passed a string constant:
//
// const str = "valid/string"
// MustParseReference(str)
// MustParseReference("another/valid/string")
// MustParseReference(str + "/and/more")
// const str = "valid/string"
// MustParseReference(str)
// MustParseReference("another/valid/string")
// MustParseReference(str + "/and/more")
//
// These will not compile:
//
// var str = "valid/string"
// MustParseReference(str)
// MustParseReference(strings.Join([]string{"valid", "string"}, "/"))
// var str = "valid/string"
// MustParseReference(str)
// MustParseReference(strings.Join([]string{"valid", "string"}, "/"))
func MustParseReference(s stringConst, opts ...Option) Reference {
ref, err := ParseReference(string(s), opts...)
if err != nil {

View File

@ -117,7 +117,7 @@ func pbDecrypt(info decryptable, password []byte) (decrypted []byte, err error)
}
ps := decrypted[len(decrypted)-psLen:]
decrypted = decrypted[:len(decrypted)-psLen]
if bytes.Compare(ps, bytes.Repeat([]byte{byte(psLen)}, psLen)) != 0 {
if !bytes.Equal(ps, bytes.Repeat([]byte{byte(psLen)}, psLen)) {
return nil, ErrDecryption
}

View File

@ -96,13 +96,13 @@ package module
// Changes to the semantics in this file require approval from rsc.
import (
"errors"
"fmt"
"path"
"sort"
"strings"
"unicode"
"unicode/utf8"
"errors"
"golang.org/x/mod/semver"
)
@ -258,7 +258,7 @@ func modPathOK(r rune) bool {
return false
}
// modPathOK reports whether r can appear in a package import path element.
// importPathOK reports whether r can appear in a package import path element.
//
// Import paths are intermediate between module paths and file paths: we allow
// disallow characters that would be confusing or ambiguous as arguments to

View File

@ -40,9 +40,10 @@
// Microsoft Azure: https://cloud.google.com/iam/docs/access-resources-azure
// OIDC identity provider: https://cloud.google.com/iam/docs/access-resources-oidc
//
// For OIDC providers, the library can retrieve OIDC tokens either from a
// local file location (file-sourced credentials) or from a local server
// (URL-sourced credentials).
// For OIDC and SAML providers, the library can retrieve tokens in three ways:
// from a local file location (file-sourced credentials), from a server
// (URL-sourced credentials), or from a local executable (executable-sourced
// credentials).
// For file-sourced credentials, a background process needs to be continuously
// refreshing the file location with a new OIDC token prior to expiration.
// For tokens with one hour lifetimes, the token needs to be updated in the file
@ -50,6 +51,11 @@
// For URL-sourced credentials, a local server needs to host a GET endpoint to
// return the OIDC token. The response can be in plain text or JSON.
// Additional required request headers can also be specified.
// For executable-sourced credentials, an application needs to be available to
// output the OIDC token and other information in a JSON format.
// For more information on how these work (and how to implement
// executable-sourced credentials), please check out:
// https://cloud.google.com/iam/docs/using-workload-identity-federation#oidc
//
// # Credentials
//

View File

@ -74,12 +74,14 @@ var (
regexp.MustCompile(`(?i)^sts\.googleapis\.com$`),
regexp.MustCompile(`(?i)^sts\.[^\.\s\/\\]+\.googleapis\.com$`),
regexp.MustCompile(`(?i)^[^\.\s\/\\]+-sts\.googleapis\.com$`),
regexp.MustCompile(`(?i)^sts-[^\.\s\/\\]+\.p\.googleapis\.com$`),
}
validImpersonateURLPatterns = []*regexp.Regexp{
regexp.MustCompile(`^[^\.\s\/\\]+\.iamcredentials\.googleapis\.com$`),
regexp.MustCompile(`^iamcredentials\.googleapis\.com$`),
regexp.MustCompile(`^iamcredentials\.[^\.\s\/\\]+\.googleapis\.com$`),
regexp.MustCompile(`^[^\.\s\/\\]+-iamcredentials\.googleapis\.com$`),
regexp.MustCompile(`^iamcredentials-[^\.\s\/\\]+\.p\.googleapis\.com$`),
}
validWorkforceAudiencePattern *regexp.Regexp = regexp.MustCompile(`//iam\.googleapis\.com/locations/[^/]+/workforcePools/`)
)

View File

@ -80,6 +80,19 @@ func (lim *Limiter) Burst() int {
return lim.burst
}
// TokensAt returns the number of tokens available at time t.
func (lim *Limiter) TokensAt(t time.Time) float64 {
lim.mu.Lock()
_, _, tokens := lim.advance(t) // does not mutute lim
lim.mu.Unlock()
return tokens
}
// Tokens returns the number of tokens available now.
func (lim *Limiter) Tokens() float64 {
return lim.TokensAt(time.Now())
}
// NewLimiter returns a new Limiter that allows events up to rate r and permits
// bursts of at most b tokens.
func NewLimiter(r Limit, b int) *Limiter {
@ -89,16 +102,16 @@ func NewLimiter(r Limit, b int) *Limiter {
}
}
// Allow is shorthand for AllowN(time.Now(), 1).
// Allow reports whether an event may happen now.
func (lim *Limiter) Allow() bool {
return lim.AllowN(time.Now(), 1)
}
// AllowN reports whether n events may happen at time now.
// AllowN reports whether n events may happen at time t.
// Use this method if you intend to drop / skip events that exceed the rate limit.
// Otherwise use Reserve or Wait.
func (lim *Limiter) AllowN(now time.Time, n int) bool {
return lim.reserveN(now, n, 0).ok
func (lim *Limiter) AllowN(t time.Time, n int) bool {
return lim.reserveN(t, n, 0).ok
}
// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
@ -131,11 +144,11 @@ const InfDuration = time.Duration(math.MaxInt64)
// before taking the reserved action. Zero duration means act immediately.
// InfDuration means the limiter cannot grant the tokens requested in this
// Reservation within the maximum wait time.
func (r *Reservation) DelayFrom(now time.Time) time.Duration {
func (r *Reservation) DelayFrom(t time.Time) time.Duration {
if !r.ok {
return InfDuration
}
delay := r.timeToAct.Sub(now)
delay := r.timeToAct.Sub(t)
if delay < 0 {
return 0
}
@ -150,7 +163,7 @@ func (r *Reservation) Cancel() {
// CancelAt indicates that the reservation holder will not perform the reserved action
// and reverses the effects of this Reservation on the rate limit as much as possible,
// considering that other reservations may have already been made.
func (r *Reservation) CancelAt(now time.Time) {
func (r *Reservation) CancelAt(t time.Time) {
if !r.ok {
return
}
@ -158,7 +171,7 @@ func (r *Reservation) CancelAt(now time.Time) {
r.lim.mu.Lock()
defer r.lim.mu.Unlock()
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(t) {
return
}
@ -170,18 +183,18 @@ func (r *Reservation) CancelAt(now time.Time) {
return
}
// advance time to now
now, _, tokens := r.lim.advance(now)
t, _, tokens := r.lim.advance(t)
// calculate new number of tokens
tokens += restoreTokens
if burst := float64(r.lim.burst); tokens > burst {
tokens = burst
}
// update state
r.lim.last = now
r.lim.last = t
r.lim.tokens = tokens
if r.timeToAct == r.lim.lastEvent {
prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
if !prevEvent.Before(now) {
if !prevEvent.Before(t) {
r.lim.lastEvent = prevEvent
}
}
@ -208,8 +221,8 @@ func (lim *Limiter) Reserve() *Reservation {
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.
func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation {
r := lim.reserveN(now, n, InfDuration)
func (lim *Limiter) ReserveN(t time.Time, n int) *Reservation {
r := lim.reserveN(t, n, InfDuration)
return &r
}
@ -234,7 +247,7 @@ func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
}
// wait is the internal implementation of WaitN.
func (lim *Limiter) wait(ctx context.Context, n int, now time.Time, newTimer func(d time.Duration) (<-chan time.Time, func() bool, func())) error {
func (lim *Limiter) wait(ctx context.Context, n int, t time.Time, newTimer func(d time.Duration) (<-chan time.Time, func() bool, func())) error {
lim.mu.Lock()
burst := lim.burst
limit := lim.limit
@ -252,15 +265,15 @@ func (lim *Limiter) wait(ctx context.Context, n int, now time.Time, newTimer fun
// Determine wait limit
waitLimit := InfDuration
if deadline, ok := ctx.Deadline(); ok {
waitLimit = deadline.Sub(now)
waitLimit = deadline.Sub(t)
}
// Reserve
r := lim.reserveN(now, n, waitLimit)
r := lim.reserveN(t, n, waitLimit)
if !r.ok {
return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n)
}
// Wait if necessary
delay := r.DelayFrom(now)
delay := r.DelayFrom(t)
if delay == 0 {
return nil
}
@ -287,13 +300,13 @@ func (lim *Limiter) SetLimit(newLimit Limit) {
// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
// before SetLimitAt was called.
func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
func (lim *Limiter) SetLimitAt(t time.Time, newLimit Limit) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
t, _, tokens := lim.advance(t)
lim.last = now
lim.last = t
lim.tokens = tokens
lim.limit = newLimit
}
@ -304,13 +317,13 @@ func (lim *Limiter) SetBurst(newBurst int) {
}
// SetBurstAt sets a new burst size for the limiter.
func (lim *Limiter) SetBurstAt(now time.Time, newBurst int) {
func (lim *Limiter) SetBurstAt(t time.Time, newBurst int) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
t, _, tokens := lim.advance(t)
lim.last = now
lim.last = t
lim.tokens = tokens
lim.burst = newBurst
}
@ -318,7 +331,7 @@ func (lim *Limiter) SetBurstAt(now time.Time, newBurst int) {
// reserveN is a helper method for AllowN, ReserveN, and WaitN.
// maxFutureReserve specifies the maximum reservation wait duration allowed.
// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
func (lim *Limiter) reserveN(t time.Time, n int, maxFutureReserve time.Duration) Reservation {
lim.mu.Lock()
defer lim.mu.Unlock()
@ -327,7 +340,7 @@ func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duratio
ok: true,
lim: lim,
tokens: n,
timeToAct: now,
timeToAct: t,
}
} else if lim.limit == 0 {
var ok bool
@ -339,11 +352,11 @@ func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duratio
ok: ok,
lim: lim,
tokens: lim.burst,
timeToAct: now,
timeToAct: t,
}
}
now, last, tokens := lim.advance(now)
t, last, tokens := lim.advance(t)
// Calculate the remaining number of tokens resulting from the request.
tokens -= float64(n)
@ -365,12 +378,12 @@ func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duratio
}
if ok {
r.tokens = n
r.timeToAct = now.Add(waitDuration)
r.timeToAct = t.Add(waitDuration)
}
// Update state
if ok {
lim.last = now
lim.last = t
lim.tokens = tokens
lim.lastEvent = r.timeToAct
} else {
@ -383,20 +396,20 @@ func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duratio
// advance calculates and returns an updated state for lim resulting from the passage of time.
// lim is not changed.
// advance requires that lim.mu is held.
func (lim *Limiter) advance(now time.Time) (newNow time.Time, newLast time.Time, newTokens float64) {
func (lim *Limiter) advance(t time.Time) (newT time.Time, newLast time.Time, newTokens float64) {
last := lim.last
if now.Before(last) {
last = now
if t.Before(last) {
last = t
}
// Calculate the new number of tokens, due to time that passed.
elapsed := now.Sub(last)
elapsed := t.Sub(last)
delta := lim.limit.tokensFromDuration(elapsed)
tokens := lim.tokens + delta
if burst := float64(lim.burst); tokens > burst {
tokens = burst
}
return now, last, tokens
return t, last, tokens
}
// durationFromTokens is a unit conversion function from the number of tokens to the duration

20
vendor/k8s.io/kube-openapi/pkg/internal/flags.go generated vendored Normal file
View File

@ -0,0 +1,20 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
// Used by tests to selectively disable experimental JSON unmarshaler
var UseOptimizedJSONUnmarshaling bool = true

View File

@ -0,0 +1,3 @@
# This source code refers to The Go Authors for copyright purposes.
# The master list of authors is in the main Go distribution,
# visible at https://tip.golang.org/AUTHORS.

View File

@ -0,0 +1,3 @@
# This source code was written by the Go contributors.
# The master list of contributors is in the main Go distribution,
# visible at https://tip.golang.org/CONTRIBUTORS.

View File

@ -0,0 +1,27 @@
Copyright (c) 2020 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -0,0 +1,321 @@
# JSON Serialization (v2)
[![GoDev](https://img.shields.io/static/v1?label=godev&message=reference&color=00add8)](https://pkg.go.dev/github.com/go-json-experiment/json)
[![Build Status](https://github.com/go-json-experiment/json/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/go-json-experiment/json/actions)
This module hosts an experimental implementation of v2 `encoding/json`.
The API is unstable and breaking changes will regularly be made.
Do not depend on this in publicly available modules.
## Goals and objectives
* **Mostly backwards compatible:** If possible, v2 should aim to be _mostly_
compatible with v1 in terms of both API and default behavior to ease migration.
For example, the `Marshal` and `Unmarshal` functions are the most widely used
declarations in the v1 package. It seems sensible for equivalent functionality
in v2 to be named the same and have the same signature.
Behaviorally, we should aim for 95% to 99% backwards compatibility.
We do not aim for 100% compatibility since we want the freedom to break
certain behaviors that are now considered to have been a mistake.
We may provide options that can bring the v2 implementation to 100% compatibility,
but it will not be the default.
* **More flexible:** There is a
[long list of feature requests](https://github.com/golang/go/issues?q=is%3Aissue+is%3Aopen+encoding%2Fjson+in%3Atitle).
We should aim to provide the most flexible features that addresses most usages.
We do not want to over fit the v2 API to handle every possible use case.
Ideally, the features provided should be orthogonal in nature such that
any combination of features results in as few surprising edge cases as possible.
* **More performant:** JSON serialization is widely used and any bit of extra
performance gains will be greatly appreciated. Some rarely used behaviors of v1
may be dropped in favor of better performance. For example,
despite `Encoder` and `Decoder` operating on an `io.Writer` and `io.Reader`,
they do not operate in a truly streaming manner,
leading to a loss in performance. The v2 implementation should aim to be truly
streaming by default (see [#33714](https://golang.org/issue/33714)).
* **Easy to use (hard to misuse):** The v2 API should aim to make
the common case easy and the less common case at least possible.
The API should avoid behavior that goes contrary to user expectation,
which may result in subtle bugs (see [#36225](https://golang.org/issue/36225)).
* **v1 and v2 maintainability:** Since the v1 implementation must stay forever,
it would be beneficial if v1 could be implemented under the hood with v2,
allowing for less maintenance burden in the future. This probably implies that
behavioral changes in v2 relative to v1 need to be exposed as options.
* **Avoid unsafe:** Standard library packages generally avoid the use of
package `unsafe` even if it could provide a performance boost.
We aim to preserve this property.
## Expectations
While this module aims to possibly be the v2 implementation of `encoding/json`,
there is no guarantee that this outcome will occur. As with any major change
to the Go standard library, this will eventually go through the
[Go proposal process](https://github.com/golang/proposal#readme).
At the present moment, this is still in the design and experimentation phase
and is not ready for a formal proposal.
There are several possible outcomes from this experiment:
1. We determine that a v2 `encoding/json` would not provide sufficient benefit
over the existing v1 `encoding/json` package. Thus, we abandon this effort.
2. We propose a v2 `encoding/json` design, but it is rejected in favor of some
other design that is considered superior.
3. We propose a v2 `encoding/json` design, but rather than adding an entirely
new v2 `encoding/json` package, we decide to merge its functionality into
the existing v1 `encoding/json` package.
4. We propose a v2 `encoding/json` design and it is accepted, resulting in
its addition to the standard library.
5. Some other unforeseen outcome (among the infinite number of possibilities).
## Development
This module is primarily developed by
[@dsnet](https://github.com/dsnet),
[@mvdan](https://github.com/mvdan), and
[@johanbrandhorst](https://github.com/johanbrandhorst)
with feedback provided by
[@rogpeppe](https://github.com/rogpeppe),
[@ChrisHines](https://github.com/ChrisHines), and
[@rsc](https://github.com/rsc).
Discussion about semantics occur semi-regularly, where a
[record of past meetings can be found here](https://docs.google.com/document/d/1rovrOTd-wTawGMPPlPuKhwXaYBg9VszTXR9AQQL5LfI/edit?usp=sharing).
## Design overview
This package aims to provide a clean separation between syntax and semantics.
Syntax deals with the structural representation of JSON (as specified in
[RFC 4627](https://tools.ietf.org/html/rfc4627),
[RFC 7159](https://tools.ietf.org/html/rfc7159),
[RFC 7493](https://tools.ietf.org/html/rfc7493),
[RFC 8259](https://tools.ietf.org/html/rfc8259), and
[RFC 8785](https://tools.ietf.org/html/rfc8785)).
Semantics deals with the meaning of syntactic data as usable application data.
The `Encoder` and `Decoder` types are streaming tokenizers concerned with the
packing or parsing of JSON data. They operate on `Token` and `RawValue` types
which represent the common data structures that are representable in JSON.
`Encoder` and `Decoder` do not aim to provide any interpretation of the data.
Functions like `Marshal`, `MarshalFull`, `MarshalNext`, `Unmarshal`,
`UnmarshalFull`, and `UnmarshalNext` provide semantic meaning by correlating
any arbitrary Go type with some JSON representation of that type (as stored in
data types like `[]byte`, `io.Writer`, `io.Reader`, `Encoder`, or `Decoder`).
![API overview](api.png)
This diagram provides a high-level overview of the v2 `json` package.
Purple blocks represent types, while blue blocks represent functions or methods.
The arrows and their direction represent the approximate flow of data.
The bottom half of the diagram contains functionality that is only concerned
with syntax, while the upper half contains functionality that assigns
semantic meaning to syntactic data handled by the bottom half.
In contrast to v1 `encoding/json`, options are represented as separate types
rather than being setter methods on the `Encoder` or `Decoder` types.
## Behavior changes
The v2 `json` package changes the default behavior of `Marshal` and `Unmarshal`
relative to the v1 `json` package to be more sensible.
Some of these behavior changes have options and workarounds to opt into
behavior similar to what v1 provided.
This table shows an overview of the changes:
| v1 | v2 | Details |
| -- | -- | ------- |
| JSON object members are unmarshaled into a Go struct using a **case-insensitive name match**. | JSON object members are unmarshaled into a Go struct using a **case-sensitive name match**. | [CaseSensitivity](/diff_test.go#:~:text=TestCaseSensitivity) |
| When marshaling a Go struct, a struct field marked as `omitempty` is omitted if **the field value is an empty Go value**, which is defined as false, 0, a nil pointer, a nil interface value, and any empty array, slice, map, or string. | When marshaling a Go struct, a struct field marked as `omitempty` is omitted if **the field value would encode as an empty JSON value**, which is defined as a JSON null, or an empty JSON string, object, or array. | [OmitEmptyOption](/diff_test.go#:~:text=TestOmitEmptyOption) |
| The `string` option **does affect** Go bools. | The `string` option **does not affect** Go bools. | [StringOption](/diff_test.go#:~:text=TestStringOption) |
| The `string` option **does not recursively affect** sub-values of the Go field value. | The `string` option **does recursively affect** sub-values of the Go field value. | [StringOption](/diff_test.go#:~:text=TestStringOption) |
| The `string` option **sometimes accepts** a JSON null escaped within a JSON string. | The `string` option **never accepts** a JSON null escaped within a JSON string. | [StringOption](/diff_test.go#:~:text=TestStringOption) |
| A nil Go slice is marshaled as a **JSON null**. | A nil Go slice is marshaled as an **empty JSON array**. | [NilSlicesAndMaps](/diff_test.go#:~:text=TestNilSlicesAndMaps) |
| A nil Go map is marshaled as a **JSON null**. | A nil Go map is marshaled as an **empty JSON object**. | [NilSlicesAndMaps](/diff_test.go#:~:text=TestNilSlicesAndMaps) |
| A Go array may be unmarshaled from a **JSON array of any length**. | A Go array must be unmarshaled from a **JSON array of the same length**. | [Arrays](/diff_test.go#:~:text=Arrays) |
| A Go byte array is represented as a **JSON array of JSON numbers**. | A Go byte array is represented as a **Base64-encoded JSON string**. | [ByteArrays](/diff_test.go#:~:text=TestByteArrays) |
| `MarshalJSON` and `UnmarshalJSON` methods declared on a pointer receiver are **inconsistently called**. | `MarshalJSON` and `UnmarshalJSON` methods declared on a pointer receiver are **consistently called**. | [PointerReceiver](/diff_test.go#:~:text=TestPointerReceiver) |
| A Go map is marshaled in a **deterministic order**. | A Go map is marshaled in a **non-deterministic order**. | [MapDeterminism](/diff_test.go#:~:text=TestMapDeterminism) |
| JSON strings are encoded **with HTML-specific characters being escaped**. | JSON strings are encoded **without any characters being escaped** (unless necessary). | [EscapeHTML](/diff_test.go#:~:text=TestEscapeHTML) |
| When marshaling, invalid UTF-8 within a Go string **are silently replaced**. | When marshaling, invalid UTF-8 within a Go string **results in an error**. | [InvalidUTF8](/diff_test.go#:~:text=TestInvalidUTF8) |
| When unmarshaling, invalid UTF-8 within a JSON string **are silently replaced**. | When unmarshaling, invalid UTF-8 within a JSON string **results in an error**. | [InvalidUTF8](/diff_test.go#:~:text=TestInvalidUTF8) |
| When marshaling, **an error does not occur** if the output JSON value contains objects with duplicate names. | When marshaling, **an error does occur** if the output JSON value contains objects with duplicate names. | [DuplicateNames](/diff_test.go#:~:text=TestDuplicateNames) |
| When unmarshaling, **an error does not occur** if the input JSON value contains objects with duplicate names. | When unmarshaling, **an error does occur** if the input JSON value contains objects with duplicate names. | [DuplicateNames](/diff_test.go#:~:text=TestDuplicateNames) |
| Unmarshaling a JSON null into a non-empty Go value **inconsistently clears the value or does nothing**. | Unmarshaling a JSON null into a non-empty Go value **always clears the value**. | [MergeNull](/diff_test.go#:~:text=TestMergeNull) |
| Unmarshaling a JSON value into a non-empty Go value **follows inconsistent and bizarre behavior**. | Unmarshaling a JSON value into a non-empty Go value **always merges if the input is an object, and otherwise replaces**. | [MergeComposite](/diff_test.go#:~:text=TestMergeComposite) |
| A `time.Duration` is represented as a **JSON number containing the decimal number of nanoseconds**. | A `time.Duration` is represented as a **JSON string containing the formatted duration (e.g., "1h2m3.456s")**. | [TimeDurations](/diff_test.go#:~:text=TestTimeDurations) |
| Unmarshaling a JSON number into a Go float beyond its representation **results in an error**. | Unmarshaling a JSON number into a Go float beyond its representation **uses the closest representable value (e.g., ±`math.MaxFloat`)**. | [MaxFloats](/diff_test.go#:~:text=TestMaxFloats) |
| A Go struct with only unexported fields **can be serialized**. | A Go struct with only unexported fields **cannot be serialized**. | [EmptyStructs](/diff_test.go#:~:text=TestEmptyStructs) |
| A Go struct that embeds an unexported struct type **can sometimes be serialized**. | A Go struct that embeds an unexported struct type **cannot be serialized**. | [EmbedUnexported](/diff_test.go#:~:text=TestEmbedUnexported) |
See [diff_test.go](/diff_test.go) for details about every change.
## Performance
One of the goals of the v2 module is to be more performant than v1.
Each of the charts below show the performance across
several different JSON implementations:
* `JSONv1` is `encoding/json` at `v1.18.2`
* `JSONv2` is `github.com/go-json-experiment/json` at `v0.0.0-20220524042235-dd8be80fc4a7`
* `JSONIterator` is `github.com/json-iterator/go` at `v1.1.12`
* `SegmentJSON` is `github.com/segmentio/encoding/json` at `v0.3.5`
* `GoJSON` is `github.com/goccy/go-json` at `v0.9.7`
* `SonicJSON` is `github.com/bytedance/sonic` at `v1.3.0`
Benchmarks were run across various datasets:
* `CanadaGeometry` is a GeoJSON (RFC 7946) representation of Canada.
It contains many JSON arrays of arrays of two-element arrays of numbers.
* `CITMCatalog` contains many JSON objects using numeric names.
* `SyntheaFHIR` is sample JSON data from the healthcare industry.
It contains many nested JSON objects with mostly string values,
where the set of unique string values is relatively small.
* `TwitterStatus` is the JSON response from the Twitter API.
It contains a mix of all different JSON kinds, where string values
are a mix of both single-byte ASCII and multi-byte Unicode.
* `GolangSource` is a simple tree representing the Go source code.
It contains many nested JSON objects, each with the same schema.
* `StringUnicode` contains many strings with multi-byte Unicode runes.
All of the implementations other than `JSONv1` and `JSONv2` make
extensive use of `unsafe`. As such, we expect those to generally be faster,
but at the cost of memory and type safety. `SonicJSON` goes a step even further
and uses just-in-time compilation to generate machine code specialized
for the Go type being marshaled or unmarshaled.
Also, `SonicJSON` does not validate JSON strings for valid UTF-8,
and so gains a notable performance boost on datasets with multi-byte Unicode.
Benchmarks are performed based on the default marshal and unmarshal behavior
of each package. Note that `JSONv2` aims to be safe and correct by default,
which may not be the most performant strategy.
`JSONv2` has several semantic changes relative to `JSONv1` that
impacts performance:
1. When marshaling, `JSONv2` no longer sorts the keys of a Go map.
This will improve performance.
2. When marshaling or unmarshaling, `JSONv2` always checks
to make sure JSON object names are unique.
This will hurt performance, but is more correct.
3. When marshaling or unmarshaling, `JSONv2` always
shallow copies the underlying value for a Go interface and
shallow copies the key and value for entries in a Go map.
This is done to keep the value as addressable so that `JSONv2` can
call methods and functions that operate on a pointer receiver.
This will hurt performance, but is more correct.
All of the charts are unit-less since the values are normalized
relative to `JSONv1`, which is why `JSONv1` always has a value of 1.
A lower value is better (i.e., runs faster).
Benchmarks were performed on an AMD Ryzen 9 5900X.
The code for the benchmarks is located at
https://github.com/go-json-experiment/jsonbench.
### Marshal Performance
#### Concrete types
![Benchmark Marshal Concrete](benchmark-marshal-concrete.png)
* This compares marshal performance when serializing
[from concrete types](/testdata_test.go).
* The `JSONv1` implementation is close to optimal (without the use of `unsafe`).
* Relative to `JSONv1`, `JSONv2` is generally as fast or slightly faster.
* Relative to `JSONIterator`, `JSONv2` is up to 1.3x faster.
* Relative to `SegmentJSON`, `JSONv2` is up to 1.8x slower.
* Relative to `GoJSON`, `JSONv2` is up to 2.0x slower.
* Relative to `SonicJSON`, `JSONv2` is about 1.8x to 3.2x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* For `JSONv1` and `JSONv2`, marshaling from concrete types is
mostly limited by the performance of Go reflection.
#### Interface types
![Benchmark Marshal Interface](benchmark-marshal-interface.png)
* This compares marshal performance when serializing from
`any`, `map[string]any`, and `[]any` types.
* Relative to `JSONv1`, `JSONv2` is about 1.5x to 4.2x faster.
* Relative to `JSONIterator`, `JSONv2` is about 1.1x to 2.4x faster.
* Relative to `SegmentJSON`, `JSONv2` is about 1.2x to 1.8x faster.
* Relative to `GoJSON`, `JSONv2` is about 1.1x to 2.5x faster.
* Relative to `SonicJSON`, `JSONv2` is up to 1.5x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* `JSONv2` is faster than the alternatives.
One advantange is because it does not sort the keys for a `map[string]any`,
while alternatives (except `SonicJSON` and `JSONIterator`) do sort the keys.
#### RawValue types
![Benchmark Marshal Rawvalue](benchmark-marshal-rawvalue.png)
* This compares performance when marshaling from a `json.RawValue`.
This mostly exercises the underlying encoder and
hides the cost of Go reflection.
* Relative to `JSONv1`, `JSONv2` is about 3.5x to 7.8x faster.
* `JSONIterator` is blazingly fast because
[it does not validate whether the raw value is valid](https://go.dev/play/p/bun9IXQCKRe)
and simply copies it to the output.
* Relative to `SegmentJSON`, `JSONv2` is about 1.5x to 2.7x faster.
* Relative to `GoJSON`, `JSONv2` is up to 2.2x faster.
* Relative to `SonicJSON`, `JSONv2` is up to 1.5x faster.
* Aside from `JSONIterator`, `JSONv2` is generally the fastest.
### Unmarshal Performance
#### Concrete types
![Benchmark Unmarshal Concrete](benchmark-unmarshal-concrete.png)
* This compares unmarshal performance when deserializing
[into concrete types](/testdata_test.go).
* Relative to `JSONv1`, `JSONv2` is about 1.8x to 5.7x faster.
* Relative to `JSONIterator`, `JSONv2` is about 1.1x to 1.6x slower.
* Relative to `SegmentJSON`, `JSONv2` is up to 2.5x slower.
* Relative to `GoJSON`, `JSONv2` is about 1.4x to 2.1x slower.
* Relative to `SonicJSON`, `JSONv2` is up to 4.0x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* For `JSONv1` and `JSONv2`, unmarshaling into concrete types is
mostly limited by the performance of Go reflection.
#### Interface types
![Benchmark Unmarshal Interface](benchmark-unmarshal-interface.png)
* This compares unmarshal performance when deserializing into
`any`, `map[string]any`, and `[]any` types.
* Relative to `JSONv1`, `JSONv2` is about 1.tx to 4.3x faster.
* Relative to `JSONIterator`, `JSONv2` is up to 1.5x faster.
* Relative to `SegmentJSON`, `JSONv2` is about 1.5 to 3.7x faster.
* Relative to `GoJSON`, `JSONv2` is up to 1.3x faster.
* Relative to `SonicJSON`, `JSONv2` is up to 1.5x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* Aside from `SonicJSON`, `JSONv2` is generally just as fast
or faster than all the alternatives.
#### RawValue types
![Benchmark Unmarshal Rawvalue](benchmark-unmarshal-rawvalue.png)
* This compares performance when unmarshaling into a `json.RawValue`.
This mostly exercises the underlying decoder and
hides away most of the cost of Go reflection.
* Relative to `JSONv1`, `JSONv2` is about 8.3x to 17.0x faster.
* Relative to `JSONIterator`, `JSONv2` is up to 2.0x faster.
* Relative to `SegmentJSON`, `JSONv2` is up to 1.6x faster or 1.7x slower.
* Relative to `GoJSON`, `JSONv2` is up to 1.9x faster or 2.1x slower.
* Relative to `SonicJSON`, `JSONv2` is up to 2.0x faster
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* `JSONv1` takes a
[lexical scanning approach](https://talks.golang.org/2011/lex.slide#1),
which performs a virtual function call for every byte of input.
In contrast, `JSONv2` makes heavy use of iterative and linear parsing logic
(with extra complexity to resume parsing when encountering segmented buffers).
* `JSONv2` is comparable to the alternatives that use `unsafe`.
Generally it is faster, but sometimes it is slower.

View File

@ -0,0 +1,506 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"io"
"reflect"
"sync"
)
// MarshalOptions configures how Go data is serialized as JSON data.
// The zero value is equivalent to the default marshal settings.
type MarshalOptions struct {
requireKeyedLiterals
nonComparable
// Marshalers is a list of type-specific marshalers to use.
Marshalers *Marshalers
// StringifyNumbers specifies that numeric Go types should be serialized
// as a JSON string containing the equivalent JSON number value.
//
// According to RFC 8259, section 6, a JSON implementation may choose to
// limit the representation of a JSON number to an IEEE 754 binary64 value.
// This may cause decoders to lose precision for int64 and uint64 types.
// Escaping JSON numbers as a JSON string preserves the exact precision.
StringifyNumbers bool
// DiscardUnknownMembers specifies that marshaling should ignore any
// JSON object members stored in Go struct fields dedicated to storing
// unknown JSON object members.
DiscardUnknownMembers bool
// formatDepth is the depth at which we respect the format flag.
formatDepth int
// format is custom formatting for the value at the specified depth.
format string
}
// Marshal serializes a Go value as a []byte with default options.
// It is a thin wrapper over MarshalOptions.Marshal.
func Marshal(in any) (out []byte, err error) {
return MarshalOptions{}.Marshal(EncodeOptions{}, in)
}
// MarshalFull serializes a Go value into an io.Writer with default options.
// It is a thin wrapper over MarshalOptions.MarshalFull.
func MarshalFull(out io.Writer, in any) error {
return MarshalOptions{}.MarshalFull(EncodeOptions{}, out, in)
}
// Marshal serializes a Go value as a []byte according to the provided
// marshal and encode options. It does not terminate the output with a newline.
// See MarshalNext for details about the conversion of a Go value into JSON.
func (mo MarshalOptions) Marshal(eo EncodeOptions, in any) (out []byte, err error) {
enc := getBufferedEncoder(eo)
defer putBufferedEncoder(enc)
enc.options.omitTopLevelNewline = true
err = mo.MarshalNext(enc, in)
// TODO(https://go.dev/issue/45038): Use bytes.Clone.
return append([]byte(nil), enc.buf...), err
}
// MarshalFull serializes a Go value into an io.Writer according to the provided
// marshal and encode options. It does not terminate the output with a newline.
// See MarshalNext for details about the conversion of a Go value into JSON.
func (mo MarshalOptions) MarshalFull(eo EncodeOptions, out io.Writer, in any) error {
enc := getStreamingEncoder(out, eo)
defer putStreamingEncoder(enc)
enc.options.omitTopLevelNewline = true
err := mo.MarshalNext(enc, in)
return err
}
// MarshalNext encodes a Go value as the next JSON value according to
// the provided marshal options.
//
// Type-specific marshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when encoding
// a value of type T (by taking its address) or a non-nil value of *T.
// MarshalNext ensures that a value is always addressable
// (by boxing it on the heap if necessary) so that
// these functions and methods can be consistently called. For performance,
// it is recommended that MarshalNext be passed a non-nil pointer to the value.
//
// The input value is encoded as JSON according the following rules:
//
// - If any type-specific functions in MarshalOptions.Marshalers match
// the value type, then those functions are called to encode the value.
// If all applicable functions return SkipFunc,
// then the value is encoded according to subsequent rules.
//
// - If the value type implements MarshalerV2,
// then the MarshalNextJSON method is called to encode the value.
//
// - If the value type implements MarshalerV1,
// then the MarshalJSON method is called to encode the value.
//
// - If the value type implements encoding.TextMarshaler,
// then the MarshalText method is called to encode the value and
// subsequently encode its result as a JSON string.
//
// - Otherwise, the value is encoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation.
// Certain types support specialized formatting according to
// a format flag optionally specified in the Go struct tag
// for the struct field that contains the current value
// (see the “JSON Representation of Go structs” section for more details).
//
// The representation of each type is as follows:
//
// - A Go boolean is encoded as a JSON boolean (e.g., true or false).
// It does not support any custom format flags.
//
// - A Go string is encoded as a JSON string.
// It does not support any custom format flags.
//
// - A Go []byte or [N]byte is encoded as a JSON string containing
// the binary value encoded using RFC 4648.
// If the format is "base64" or unspecified, then this uses RFC 4648, section 4.
// If the format is "base64url", then this uses RFC 4648, section 5.
// If the format is "base32", then this uses RFC 4648, section 6.
// If the format is "base32hex", then this uses RFC 4648, section 7.
// If the format is "base16" or "hex", then this uses RFC 4648, section 8.
// If the format is "array", then the bytes value is encoded as a JSON array
// where each byte is recursively JSON-encoded as each JSON array element.
//
// - A Go integer is encoded as a JSON number without fractions or exponents.
// If MarshalOptions.StringifyNumbers is specified, then the JSON number is
// encoded within a JSON string. It does not support any custom format
// flags.
//
// - A Go float is encoded as a JSON number.
// If MarshalOptions.StringifyNumbers is specified,
// then the JSON number is encoded within a JSON string.
// If the format is "nonfinite", then NaN, +Inf, and -Inf are encoded as
// the JSON strings "NaN", "Infinity", and "-Infinity", respectively.
// Otherwise, the presence of non-finite numbers results in a SemanticError.
//
// - A Go map is encoded as a JSON object, where each Go map key and value
// is recursively encoded as a name and value pair in the JSON object.
// The Go map key must encode as a JSON string, otherwise this results
// in a SemanticError. When encoding keys, MarshalOptions.StringifyNumbers
// is automatically applied so that numeric keys encode as JSON strings.
// The Go map is traversed in a non-deterministic order.
// For deterministic encoding, consider using RawValue.Canonicalize.
// If the format is "emitnull", then a nil map is encoded as a JSON null.
// Otherwise by default, a nil map is encoded as an empty JSON object.
//
// - A Go struct is encoded as a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is encoded as a JSON array, where each Go slice element
// is recursively JSON-encoded as the elements of the JSON array.
// If the format is "emitnull", then a nil slice is encoded as a JSON null.
// Otherwise by default, a nil slice is encoded as an empty JSON array.
//
// - A Go array is encoded as a JSON array, where each Go array element
// is recursively JSON-encoded as the elements of the JSON array.
// The JSON array length is always identical to the Go array length.
// It does not support any custom format flags.
//
// - A Go pointer is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
// Format flags are forwarded to the encoding of the underlying value.
//
// - A Go interface is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
// It does not support any custom format flags.
//
// - A Go time.Time is encoded as a JSON string containing the timestamp
// formatted in RFC 3339 with nanosecond resolution.
// If the format matches one of the format constants declared
// in the time package (e.g., RFC1123), then that format is used.
// Otherwise, the format is used as-is with time.Time.Format if non-empty.
//
// - A Go time.Duration is encoded as a JSON string containing the duration
// formatted according to time.Duration.String.
// If the format is "nanos", it is encoded as a JSON number
// containing the number of nanoseconds in the duration.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a SemanticError.
//
// JSON cannot represent cyclic data structures and
// MarshalNext does not handle them.
// Passing cyclic structures will result in an error.
func (mo MarshalOptions) MarshalNext(out *Encoder, in any) error {
v := reflect.ValueOf(in)
if !v.IsValid() || (v.Kind() == reflect.Pointer && v.IsNil()) {
return out.WriteToken(Null)
}
// Shallow copy non-pointer values to obtain an addressable value.
// It is beneficial to performance to always pass pointers to avoid this.
if v.Kind() != reflect.Pointer {
v2 := reflect.New(v.Type())
v2.Elem().Set(v)
v = v2
}
va := addressableValue{v.Elem()} // dereferenced pointer is always addressable
t := va.Type()
// Lookup and call the marshal function for this type.
marshal := lookupArshaler(t).marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.lookup(marshal, t)
}
if err := marshal(mo, out, va); err != nil {
if !out.options.AllowDuplicateNames {
out.tokens.invalidateDisabledNamespaces()
}
return err
}
return nil
}
// UnmarshalOptions configures how JSON data is deserialized as Go data.
// The zero value is equivalent to the default unmarshal settings.
type UnmarshalOptions struct {
requireKeyedLiterals
nonComparable
// Unmarshalers is a list of type-specific unmarshalers to use.
Unmarshalers *Unmarshalers
// StringifyNumbers specifies that numeric Go types can be deserialized
// from either a JSON number or a JSON string containing a JSON number
// without any surrounding whitespace.
StringifyNumbers bool
// RejectUnknownMembers specifies that unknown members should be rejected
// when unmarshaling a JSON object, regardless of whether there is a field
// to store unknown members.
RejectUnknownMembers bool
// formatDepth is the depth at which we respect the format flag.
formatDepth int
// format is custom formatting for the value at the specified depth.
format string
}
// Unmarshal deserializes a Go value from a []byte with default options.
// It is a thin wrapper over UnmarshalOptions.Unmarshal.
func Unmarshal(in []byte, out any) error {
return UnmarshalOptions{}.Unmarshal(DecodeOptions{}, in, out)
}
// UnmarshalFull deserializes a Go value from an io.Reader with default options.
// It is a thin wrapper over UnmarshalOptions.UnmarshalFull.
func UnmarshalFull(in io.Reader, out any) error {
return UnmarshalOptions{}.UnmarshalFull(DecodeOptions{}, in, out)
}
// Unmarshal deserializes a Go value from a []byte according to the
// provided unmarshal and decode options. The output must be a non-nil pointer.
// The input must be a single JSON value with optional whitespace interspersed.
// See UnmarshalNext for details about the conversion of JSON into a Go value.
func (uo UnmarshalOptions) Unmarshal(do DecodeOptions, in []byte, out any) error {
dec := getBufferedDecoder(in, do)
defer putBufferedDecoder(dec)
return uo.unmarshalFull(dec, out)
}
// UnmarshalFull deserializes a Go value from an io.Reader according to the
// provided unmarshal and decode options. The output must be a non-nil pointer.
// The input must be a single JSON value with optional whitespace interspersed.
// It consumes the entirety of io.Reader until io.EOF is encountered.
// See UnmarshalNext for details about the conversion of JSON into a Go value.
func (uo UnmarshalOptions) UnmarshalFull(do DecodeOptions, in io.Reader, out any) error {
dec := getStreamingDecoder(in, do)
defer putStreamingDecoder(dec)
return uo.unmarshalFull(dec, out)
}
func (uo UnmarshalOptions) unmarshalFull(in *Decoder, out any) error {
switch err := uo.UnmarshalNext(in, out); err {
case nil:
return in.checkEOF()
case io.EOF:
return io.ErrUnexpectedEOF
default:
return err
}
}
// UnmarshalNext decodes the next JSON value into a Go value according to
// the provided unmarshal options. The output must be a non-nil pointer.
//
// Type-specific unmarshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when decoding
// a value of type T (by taking its address) or a non-nil value of *T.
// UnmarshalNext ensures that a value is always addressable
// (by boxing it on the heap if necessary) so that
// these functions and methods can be consistently called.
//
// The input is decoded into the output according the following rules:
//
// - If any type-specific functions in UnmarshalOptions.Unmarshalers match
// the value type, then those functions are called to decode the JSON
// value. If all applicable functions return SkipFunc,
// then the input is decoded according to subsequent rules.
//
// - If the value type implements UnmarshalerV2,
// then the UnmarshalNextJSON method is called to decode the JSON value.
//
// - If the value type implements UnmarshalerV1,
// then the UnmarshalJSON method is called to decode the JSON value.
//
// - If the value type implements encoding.TextUnmarshaler,
// then the input is decoded as a JSON string and
// the UnmarshalText method is called with the decoded string value.
// This fails with a SemanticError if the input is not a JSON string.
//
// - Otherwise, the JSON value is decoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation.
// Certain types support specialized formatting according to
// a format flag optionally specified in the Go struct tag
// for the struct field that contains the current value
// (see the “JSON Representation of Go structs” section for more details).
// A JSON null may be decoded into every supported Go value where
// it is equivalent to storing the zero value of the Go value.
// If the input JSON kind is not handled by the current Go value type,
// then this fails with a SemanticError. Unless otherwise specified,
// the decoded value replaces any pre-existing value.
//
// The representation of each type is as follows:
//
// - A Go boolean is decoded from a JSON boolean (e.g., true or false).
// It does not support any custom format flags.
//
// - A Go string is decoded from a JSON string.
// It does not support any custom format flags.
//
// - A Go []byte or [N]byte is decoded from a JSON string
// containing the binary value encoded using RFC 4648.
// If the format is "base64" or unspecified, then this uses RFC 4648, section 4.
// If the format is "base64url", then this uses RFC 4648, section 5.
// If the format is "base32", then this uses RFC 4648, section 6.
// If the format is "base32hex", then this uses RFC 4648, section 7.
// If the format is "base16" or "hex", then this uses RFC 4648, section 8.
// If the format is "array", then the Go slice or array is decoded from a
// JSON array where each JSON element is recursively decoded for each byte.
// When decoding into a non-nil []byte, the slice length is reset to zero
// and the decoded input is appended to it.
// When decoding into a [N]byte, the input must decode to exactly N bytes,
// otherwise it fails with a SemanticError.
//
// - A Go integer is decoded from a JSON number.
// It may also be decoded from a JSON string containing a JSON number
// if UnmarshalOptions.StringifyNumbers is specified.
// It fails with a SemanticError if the JSON number
// has a fractional or exponent component.
// It also fails if it overflows the representation of the Go integer type.
// It does not support any custom format flags.
//
// - A Go float is decoded from a JSON number.
// It may also be decoded from a JSON string containing a JSON number
// if UnmarshalOptions.StringifyNumbers is specified.
// The JSON number is parsed as the closest representable Go float value.
// If the format is "nonfinite", then the JSON strings
// "NaN", "Infinity", and "-Infinity" are decoded as NaN, +Inf, and -Inf.
// Otherwise, the presence of such strings results in a SemanticError.
//
// - A Go map is decoded from a JSON object,
// where each JSON object name and value pair is recursively decoded
// as the Go map key and value. When decoding keys,
// UnmarshalOptions.StringifyNumbers is automatically applied so that
// numeric keys can decode from JSON strings. Maps are not cleared.
// If the Go map is nil, then a new map is allocated to decode into.
// If the decoded key matches an existing Go map entry, the entry value
// is reused by decoding the JSON object value into it.
// The only supported format is "emitnull" and has no effect when decoding.
//
// - A Go struct is decoded from a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is decoded from a JSON array, where each JSON element
// is recursively decoded and appended to the Go slice.
// Before appending into a Go slice, a new slice is allocated if it is nil,
// otherwise the slice length is reset to zero.
// The only supported format is "emitnull" and has no effect when decoding.
//
// - A Go array is decoded from a JSON array, where each JSON array element
// is recursively decoded as each corresponding Go array element.
// Each Go array element is zeroed before decoding into it.
// It fails with a SemanticError if the JSON array does not contain
// the exact same number of elements as the Go array.
// It does not support any custom format flags.
//
// - A Go pointer is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil pointer.
// Otherwise, it allocates a new underlying value if the pointer is nil,
// and recursively JSON decodes into the underlying value.
// Format flags are forwarded to the decoding of the underlying type.
//
// - A Go interface is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil interface value.
// Otherwise, a nil interface value of an empty interface type is initialized
// with a zero Go bool, string, float64, map[string]any, or []any if the
// input is a JSON boolean, string, number, object, or array, respectively.
// If the interface value is still nil, then this fails with a SemanticError
// since decoding could not determine an appropriate Go type to decode into.
// For example, unmarshaling into a nil io.Reader fails since
// there is no concrete type to populate the interface value with.
// Otherwise an underlying value exists and it recursively decodes
// the JSON input into it. It does not support any custom format flags.
//
// - A Go time.Time is decoded from a JSON string containing the time
// formatted in RFC 3339 with nanosecond resolution.
// If the format matches one of the format constants declared in
// the time package (e.g., RFC1123), then that format is used for parsing.
// Otherwise, the format is used as-is with time.Time.Parse if non-empty.
//
// - A Go time.Duration is decoded from a JSON string by
// passing the decoded string to time.ParseDuration.
// If the format is "nanos", it is instead decoded from a JSON number
// containing the number of nanoseconds in the duration.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a SemanticError.
//
// In general, unmarshaling follows merge semantics (similar to RFC 7396)
// where the decoded Go value replaces the destination value
// for any JSON kind other than an object.
// For JSON objects, the input object is merged into the destination value
// where matching object members recursively apply merge semantics.
func (uo UnmarshalOptions) UnmarshalNext(in *Decoder, out any) error {
v := reflect.ValueOf(out)
if !v.IsValid() || v.Kind() != reflect.Pointer || v.IsNil() {
var t reflect.Type
if v.IsValid() {
t = v.Type()
if t.Kind() == reflect.Pointer {
t = t.Elem()
}
}
err := errors.New("value must be passed as a non-nil pointer reference")
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
va := addressableValue{v.Elem()} // dereferenced pointer is always addressable
t := va.Type()
// Lookup and call the unmarshal function for this type.
unmarshal := lookupArshaler(t).unmarshal
if uo.Unmarshalers != nil {
unmarshal, _ = uo.Unmarshalers.lookup(unmarshal, t)
}
if err := unmarshal(uo, in, va); err != nil {
if !in.options.AllowDuplicateNames {
in.tokens.invalidateDisabledNamespaces()
}
return err
}
return nil
}
// addressableValue is a reflect.Value that is guaranteed to be addressable
// such that calling the Addr and Set methods do not panic.
//
// There is no compile magic that enforces this property,
// but rather the need to construct this type makes it easier to examine each
// construction site to ensure that this property is upheld.
type addressableValue struct{ reflect.Value }
// newAddressableValue constructs a new addressable value of type t.
func newAddressableValue(t reflect.Type) addressableValue {
return addressableValue{reflect.New(t).Elem()}
}
// All marshal and unmarshal behavior is implemented using these signatures.
type (
marshaler = func(MarshalOptions, *Encoder, addressableValue) error
unmarshaler = func(UnmarshalOptions, *Decoder, addressableValue) error
)
type arshaler struct {
marshal marshaler
unmarshal unmarshaler
nonDefault bool
}
var lookupArshalerCache sync.Map // map[reflect.Type]*arshaler
func lookupArshaler(t reflect.Type) *arshaler {
if v, ok := lookupArshalerCache.Load(t); ok {
return v.(*arshaler)
}
fncs := makeDefaultArshaler(t)
fncs = makeMethodArshaler(fncs, t)
fncs = makeTimeArshaler(fncs, t)
// Use the last stored so that duplicate arshalers can be garbage collected.
v, _ := lookupArshalerCache.LoadOrStore(t, fncs)
return v.(*arshaler)
}

View File

@ -0,0 +1,219 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import "reflect"
// This files contains an optimized marshal and unmarshal implementation
// for the any type. This type is often used when the Go program has
// no knowledge of the JSON schema. This is a common enough occurrence
// to justify the complexity of adding logic for this.
func marshalValueAny(mo MarshalOptions, enc *Encoder, val any) error {
switch val := val.(type) {
case nil:
return enc.WriteToken(Null)
case bool:
return enc.WriteToken(Bool(val))
case string:
return enc.WriteToken(String(val))
case float64:
return enc.WriteToken(Float(val))
case map[string]any:
return marshalObjectAny(mo, enc, val)
case []any:
return marshalArrayAny(mo, enc, val)
default:
v := newAddressableValue(reflect.TypeOf(val))
v.Set(reflect.ValueOf(val))
marshal := lookupArshaler(v.Type()).marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.lookup(marshal, v.Type())
}
return marshal(mo, enc, v)
}
}
func unmarshalValueAny(uo UnmarshalOptions, dec *Decoder) (any, error) {
switch k := dec.PeekKind(); k {
case '{':
return unmarshalObjectAny(uo, dec)
case '[':
return unmarshalArrayAny(uo, dec)
default:
var flags valueFlags
val, err := dec.readValue(&flags)
if err != nil {
return nil, err
}
switch val.Kind() {
case 'n':
return nil, nil
case 'f':
return false, nil
case 't':
return true, nil
case '"':
val = unescapeStringMayCopy(val, flags.isVerbatim())
if dec.stringCache == nil {
dec.stringCache = new(stringCache)
}
return dec.stringCache.make(val), nil
case '0':
fv, _ := parseFloat(val, 64) // ignore error since readValue gaurantees val is valid
return fv, nil
default:
panic("BUG: invalid kind: " + k.String())
}
}
}
func marshalObjectAny(mo MarshalOptions, enc *Encoder, obj map[string]any) error {
// Check for cycles.
if enc.tokens.depth() > startDetectingCyclesAfter {
v := reflect.ValueOf(obj)
if err := enc.seenPointers.visit(v); err != nil {
return err
}
defer enc.seenPointers.leave(v)
}
// Optimize for marshaling an empty map without any preceding whitespace.
if len(obj) == 0 && !enc.options.multiline && !enc.tokens.last.needObjectName() {
enc.buf = enc.tokens.mayAppendDelim(enc.buf, '{')
enc.buf = append(enc.buf, "{}"...)
enc.tokens.last.increment()
if enc.needFlush() {
return enc.flush()
}
return nil
}
if err := enc.WriteToken(ObjectStart); err != nil {
return err
}
// A Go map guarantees that each entry has a unique key
// The only possibility of duplicates is due to invalid UTF-8.
if !enc.options.AllowInvalidUTF8 {
enc.tokens.last.disableNamespace()
}
for name, val := range obj {
if err := enc.WriteToken(String(name)); err != nil {
return err
}
if err := marshalValueAny(mo, enc, val); err != nil {
return err
}
}
if err := enc.WriteToken(ObjectEnd); err != nil {
return err
}
return nil
}
func unmarshalObjectAny(uo UnmarshalOptions, dec *Decoder) (map[string]any, error) {
tok, err := dec.ReadToken()
if err != nil {
return nil, err
}
k := tok.Kind()
switch k {
case 'n':
return nil, nil
case '{':
obj := make(map[string]any)
// A Go map guarantees that each entry has a unique key
// The only possibility of duplicates is due to invalid UTF-8.
if !dec.options.AllowInvalidUTF8 {
dec.tokens.last.disableNamespace()
}
for dec.PeekKind() != '}' {
tok, err := dec.ReadToken()
if err != nil {
return obj, err
}
name := tok.String()
// Manually check for duplicate names.
if _, ok := obj[name]; ok {
name := dec.previousBuffer()
err := &SyntacticError{str: "duplicate name " + string(name) + " in object"}
return obj, err.withOffset(dec.InputOffset() - int64(len(name)))
}
val, err := unmarshalValueAny(uo, dec)
obj[name] = val
if err != nil {
return obj, err
}
}
if _, err := dec.ReadToken(); err != nil {
return obj, err
}
return obj, nil
}
return nil, &SemanticError{action: "unmarshal", JSONKind: k, GoType: mapStringAnyType}
}
func marshalArrayAny(mo MarshalOptions, enc *Encoder, arr []any) error {
// Check for cycles.
if enc.tokens.depth() > startDetectingCyclesAfter {
v := reflect.ValueOf(arr)
if err := enc.seenPointers.visit(v); err != nil {
return err
}
defer enc.seenPointers.leave(v)
}
// Optimize for marshaling an empty slice without any preceding whitespace.
if len(arr) == 0 && !enc.options.multiline && !enc.tokens.last.needObjectName() {
enc.buf = enc.tokens.mayAppendDelim(enc.buf, '[')
enc.buf = append(enc.buf, "[]"...)
enc.tokens.last.increment()
if enc.needFlush() {
return enc.flush()
}
return nil
}
if err := enc.WriteToken(ArrayStart); err != nil {
return err
}
for _, val := range arr {
if err := marshalValueAny(mo, enc, val); err != nil {
return err
}
}
if err := enc.WriteToken(ArrayEnd); err != nil {
return err
}
return nil
}
func unmarshalArrayAny(uo UnmarshalOptions, dec *Decoder) ([]any, error) {
tok, err := dec.ReadToken()
if err != nil {
return nil, err
}
k := tok.Kind()
switch k {
case 'n':
return nil, nil
case '[':
arr := []any{}
for dec.PeekKind() != ']' {
val, err := unmarshalValueAny(uo, dec)
arr = append(arr, val)
if err != nil {
return arr, err
}
}
if _, err := dec.ReadToken(); err != nil {
return arr, err
}
return arr, nil
}
return nil, &SemanticError{action: "unmarshal", JSONKind: k, GoType: sliceAnyType}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,387 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"fmt"
"reflect"
"sync"
)
// SkipFunc may be returned by MarshalFuncV2 and UnmarshalFuncV2 functions.
//
// Any function that returns SkipFunc must not cause observable side effects
// on the provided Encoder or Decoder. For example, it is permissible to call
// Decoder.PeekKind, but not permissible to call Decoder.ReadToken or
// Encoder.WriteToken since such methods mutate the state.
const SkipFunc = jsonError("skip function")
// Marshalers is a list of functions that may override the marshal behavior
// of specific types. Populate MarshalOptions.Marshalers to use it.
// A nil *Marshalers is equivalent to an empty list.
type Marshalers = typedMarshalers
// NewMarshalers constructs a flattened list of marshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns SkipFunc, then the next applicable function is called,
// otherwise the default marshaling behavior is used.
//
// For example:
//
// m1 := NewMarshalers(f1, f2)
// m2 := NewMarshalers(f0, m1, f3) // equivalent to m3
// m3 := NewMarshalers(f0, f1, f2, f3) // equivalent to m2
func NewMarshalers(ms ...*Marshalers) *Marshalers {
return newMarshalers(ms...)
}
// Unmarshalers is a list of functions that may override the unmarshal behavior
// of specific types. Populate UnmarshalOptions.Unmarshalers to use it.
// A nil *Unmarshalers is equivalent to an empty list.
type Unmarshalers = typedUnmarshalers
// NewUnmarshalers constructs a flattened list of unmarshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns SkipFunc, then the next applicable function is called,
// otherwise the default unmarshaling behavior is used.
//
// For example:
//
// u1 := NewUnmarshalers(f1, f2)
// u2 := NewUnmarshalers(f0, u1, f3) // equivalent to u3
// u3 := NewUnmarshalers(f0, f1, f2, f3) // equivalent to u2
func NewUnmarshalers(us ...*Unmarshalers) *Unmarshalers {
return newUnmarshalers(us...)
}
type typedMarshalers = typedArshalers[MarshalOptions, Encoder]
type typedUnmarshalers = typedArshalers[UnmarshalOptions, Decoder]
type typedArshalers[Options, Coder any] struct {
nonComparable
fncVals []typedArshaler[Options, Coder]
fncCache sync.Map // map[reflect.Type]arshaler
// fromAny reports whether any of Go types used to represent arbitrary JSON
// (i.e., any, bool, string, float64, map[string]any, or []any) matches
// any of the provided type-specific arshalers.
//
// This bit of information is needed in arshal_default.go to determine
// whether to use the specialized logic in arshal_any.go to handle
// the any interface type. The logic in arshal_any.go does not support
// type-specific arshal functions, so we must avoid using that logic
// if this is true.
fromAny bool
}
type typedMarshaler = typedArshaler[MarshalOptions, Encoder]
type typedUnmarshaler = typedArshaler[UnmarshalOptions, Decoder]
type typedArshaler[Options, Coder any] struct {
typ reflect.Type
fnc func(Options, *Coder, addressableValue) error
maySkip bool
}
func newMarshalers(ms ...*Marshalers) *Marshalers { return newTypedArshalers(ms...) }
func newUnmarshalers(us ...*Unmarshalers) *Unmarshalers { return newTypedArshalers(us...) }
func newTypedArshalers[Options, Coder any](as ...*typedArshalers[Options, Coder]) *typedArshalers[Options, Coder] {
var a typedArshalers[Options, Coder]
for _, a2 := range as {
if a2 != nil {
a.fncVals = append(a.fncVals, a2.fncVals...)
a.fromAny = a.fromAny || a2.fromAny
}
}
if len(a.fncVals) == 0 {
return nil
}
return &a
}
func (a *typedArshalers[Options, Coder]) lookup(fnc func(Options, *Coder, addressableValue) error, t reflect.Type) (func(Options, *Coder, addressableValue) error, bool) {
if a == nil {
return fnc, false
}
if v, ok := a.fncCache.Load(t); ok {
if v == nil {
return fnc, false
}
return v.(func(Options, *Coder, addressableValue) error), true
}
// Collect a list of arshalers that can be called for this type.
// This list may be longer than 1 since some arshalers can be skipped.
var fncs []func(Options, *Coder, addressableValue) error
for _, fncVal := range a.fncVals {
if !castableTo(t, fncVal.typ) {
continue
}
fncs = append(fncs, fncVal.fnc)
if !fncVal.maySkip {
break // subsequent arshalers will never be called
}
}
if len(fncs) == 0 {
a.fncCache.Store(t, nil) // nil to indicate that no funcs found
return fnc, false
}
// Construct an arshaler that may call every applicable arshaler.
fncDefault := fnc
fnc = func(o Options, c *Coder, v addressableValue) error {
for _, fnc := range fncs {
if err := fnc(o, c, v); err != SkipFunc {
return err // may be nil or non-nil
}
}
return fncDefault(o, c, v)
}
// Use the first stored so duplicate work can be garbage collected.
v, _ := a.fncCache.LoadOrStore(t, fnc)
return v.(func(Options, *Coder, addressableValue) error), true
}
// MarshalFuncV1 constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value.
// The value of T must not be retained outside the function call.
// It may not return SkipFunc.
func MarshalFuncV1[T any](fn func(T) ([]byte, error)) *Marshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, true)
typFnc := typedMarshaler{
typ: t,
fnc: func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
val, err := fn(va.castTo(t).Interface().(T))
if err != nil {
err = wrapSkipFunc(err, "marshal function of type func(T) ([]byte, error)")
// TODO: Avoid wrapping semantic errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
if err := enc.WriteValue(val); err != nil {
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", JSONKind: RawValue(val).Kind(), GoType: t, Err: err}
}
return nil
},
}
return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// MarshalFuncV2 constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value by calling write methods
// on the provided encoder. It may return SkipFunc such that marshaling can
// move on to the next marshal function. However, no mutable method calls may
// be called on the encoder if SkipFunc is returned.
// The pointer to Encoder and the value of T must not be retained
// outside the function call.
func MarshalFuncV2[T any](fn func(MarshalOptions, *Encoder, T) error) *Marshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, true)
typFnc := typedMarshaler{
typ: t,
fnc: func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
prevDepth, prevLength := enc.tokens.depthLength()
err := fn(mo, enc, va.castTo(t).Interface().(T))
currDepth, currLength := enc.tokens.depthLength()
if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) {
err = errors.New("must write exactly one JSON value")
}
if err != nil {
if err == SkipFunc {
if prevDepth == currDepth && prevLength == currLength {
return SkipFunc
}
err = errors.New("must not write any JSON tokens when skipping")
}
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
return nil
},
maySkip: true,
}
return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// UnmarshalFuncV1 constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value.
// The input []byte must not be mutated.
// The input []byte and value T must not be retained outside the function call.
// It may not return SkipFunc.
func UnmarshalFuncV1[T any](fn func([]byte, T) error) *Unmarshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, false)
typFnc := typedUnmarshaler{
typ: t,
fnc: func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
val, err := dec.ReadValue()
if err != nil {
return err // must be a syntactic or I/O error
}
err = fn(val, va.castTo(t).Interface().(T))
if err != nil {
err = wrapSkipFunc(err, "unmarshal function of type func([]byte, T) error")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
return nil
},
}
return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// UnmarshalFuncV2 constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value by calling read methods
// on the provided decoder. It may return SkipFunc such that unmarshaling can
// move on to the next unmarshal function. However, no mutable method calls may
// be called on the decoder if SkipFunc is returned.
// The pointer to Decoder and the value of T must not be retained
// outside the function call.
func UnmarshalFuncV2[T any](fn func(UnmarshalOptions, *Decoder, T) error) *Unmarshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, false)
typFnc := typedUnmarshaler{
typ: t,
fnc: func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
prevDepth, prevLength := dec.tokens.depthLength()
err := fn(uo, dec, va.castTo(t).Interface().(T))
currDepth, currLength := dec.tokens.depthLength()
if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) {
err = errors.New("must read exactly one JSON value")
}
if err != nil {
if err == SkipFunc {
if prevDepth == currDepth && prevLength == currLength {
return SkipFunc
}
err = errors.New("must not read any JSON tokens when skipping")
}
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
return nil
},
maySkip: true,
}
return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// assertCastableTo asserts that "to" is a valid type to be casted to.
// These are the Go types that type-specific arshalers may operate upon.
//
// Let AllTypes be the universal set of all possible Go types.
// This function generally asserts that:
//
// len([from for from in AllTypes if castableTo(from, to)]) > 0
//
// otherwise it panics.
//
// As a special-case if marshal is false, then we forbid any non-pointer or
// non-interface type since it is almost always a bug trying to unmarshal
// into something where the end-user caller did not pass in an addressable value
// since they will not observe the mutations.
func assertCastableTo(to reflect.Type, marshal bool) {
switch to.Kind() {
case reflect.Interface:
return
case reflect.Pointer:
// Only allow unnamed pointers to be consistent with the fact that
// taking the address of a value produces an unnamed pointer type.
if to.Name() == "" {
return
}
default:
// Technically, non-pointer types are permissible for unmarshal.
// However, they are often a bug since the receiver would be immutable.
// Thus, only allow them for marshaling.
if marshal {
return
}
}
if marshal {
panic(fmt.Sprintf("input type %v must be an interface type, an unnamed pointer type, or a non-pointer type", to))
} else {
panic(fmt.Sprintf("input type %v must be an interface type or an unnamed pointer type", to))
}
}
// castableTo checks whether values of type "from" can be casted to type "to".
// Nil pointer or interface "from" values are never considered castable.
//
// This function must be kept in sync with addressableValue.castTo.
func castableTo(from, to reflect.Type) bool {
switch to.Kind() {
case reflect.Interface:
// TODO: This breaks when ordinary interfaces can have type sets
// since interfaces now exist where only the value form of a type (T)
// implements the interface, but not the pointer variant (*T).
// See https://go.dev/issue/45346.
return reflect.PointerTo(from).Implements(to)
case reflect.Pointer:
// Common case for unmarshaling.
// From must be a concrete or interface type.
return reflect.PointerTo(from) == to
default:
// Common case for marshaling.
// From must be a concrete type.
return from == to
}
}
// castTo casts va to the specified type.
// If the type is an interface, then the underlying type will always
// be a non-nil pointer to a concrete type.
//
// Requirement: castableTo(va.Type(), to) must hold.
func (va addressableValue) castTo(to reflect.Type) reflect.Value {
switch to.Kind() {
case reflect.Interface:
return va.Addr().Convert(to)
case reflect.Pointer:
return va.Addr()
default:
return va.Value
}
}
// castableToFromAny reports whether "to" can be casted to from any
// of the dynamic types used to represent arbitrary JSON.
func castableToFromAny(to reflect.Type) bool {
for _, from := range []reflect.Type{anyType, boolType, stringType, float64Type, mapStringAnyType, sliceAnyType} {
if castableTo(from, to) {
return true
}
}
return false
}
func wrapSkipFunc(err error, what string) error {
if err == SkipFunc {
return errors.New(what + " cannot be skipped")
}
return err
}

View File

@ -0,0 +1,186 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"reflect"
)
// This package supports "inlining" a Go struct field, where the contents
// of the serialized field (which must be a JSON object) are treated as if
// they are part of the parent Go struct (which represents a JSON object).
//
// Generally, inlined fields are of a Go struct type, where the fields of the
// nested struct are virtually hoisted up to the parent struct using rules
// similar to how Go embedding works (but operating within the JSON namespace).
//
// However, inlined fields may also be of a Go map type with a string key
// or a RawValue. Such inlined fields are called "fallback" fields since they
// represent any arbitrary JSON object member. Explicitly named fields take
// precedence over the inlined fallback. Only one inlined fallback is allowed.
var rawValueType = reflect.TypeOf((*RawValue)(nil)).Elem()
// marshalInlinedFallbackAll marshals all the members in an inlined fallback.
func marshalInlinedFallbackAll(mo MarshalOptions, enc *Encoder, va addressableValue, f *structField, insertUnquotedName func([]byte) bool) error {
v := addressableValue{va.Field(f.index[0])} // addressable if struct value is addressable
if len(f.index) > 1 {
v = v.fieldByIndex(f.index[1:], false)
if !v.IsValid() {
return nil // implies a nil inlined field
}
}
v = v.indirect(false)
if !v.IsValid() {
return nil
}
if v.Type() == rawValueType {
b := v.Interface().(RawValue)
if len(b) == 0 { // TODO: Should this be nil? What if it were all whitespace?
return nil
}
dec := getBufferedDecoder(b, DecodeOptions{AllowDuplicateNames: true, AllowInvalidUTF8: true})
defer putBufferedDecoder(dec)
tok, err := dec.ReadToken()
if err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if tok.Kind() != '{' {
err := errors.New("inlined raw value must be a JSON object")
return &SemanticError{action: "marshal", JSONKind: tok.Kind(), GoType: rawValueType, Err: err}
}
for dec.PeekKind() != '}' {
// Parse the JSON object name.
var flags valueFlags
val, err := dec.readValue(&flags)
if err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if insertUnquotedName != nil {
name := unescapeStringMayCopy(val, flags.isVerbatim())
if !insertUnquotedName(name) {
return &SyntacticError{str: "duplicate name " + string(val) + " in object"}
}
}
if err := enc.WriteValue(val); err != nil {
return err
}
// Parse the JSON object value.
val, err = dec.readValue(&flags)
if err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if err := enc.WriteValue(val); err != nil {
return err
}
}
if _, err := dec.ReadToken(); err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if err := dec.checkEOF(); err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
return nil
} else {
if v.Len() == 0 {
return nil
}
m := v
mv := newAddressableValue(m.Type().Elem())
for iter := m.MapRange(); iter.Next(); {
b, err := appendString(enc.UnusedBuffer(), iter.Key().String(), !enc.options.AllowInvalidUTF8, nil)
if err != nil {
return err
}
if insertUnquotedName != nil {
isVerbatim := consumeSimpleString(b) == len(b)
name := unescapeStringMayCopy(b, isVerbatim)
if !insertUnquotedName(name) {
return &SyntacticError{str: "duplicate name " + string(b) + " in object"}
}
}
if err := enc.WriteValue(b); err != nil {
return err
}
mv.Set(iter.Value())
marshal := f.fncs.marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.lookup(marshal, mv.Type())
}
if err := marshal(mo, enc, mv); err != nil {
return err
}
}
return nil
}
}
// unmarshalInlinedFallbackNext unmarshals only the next member in an inlined fallback.
func unmarshalInlinedFallbackNext(uo UnmarshalOptions, dec *Decoder, va addressableValue, f *structField, quotedName, unquotedName []byte) error {
v := addressableValue{va.Field(f.index[0])} // addressable if struct value is addressable
if len(f.index) > 1 {
v = v.fieldByIndex(f.index[1:], true)
}
v = v.indirect(true)
if v.Type() == rawValueType {
b := v.Addr().Interface().(*RawValue)
if len(*b) == 0 { // TODO: Should this be nil? What if it were all whitespace?
*b = append(*b, '{')
} else {
*b = trimSuffixWhitespace(*b)
if hasSuffixByte(*b, '}') {
// TODO: When merging into an object for the first time,
// should we verify that it is valid?
*b = trimSuffixByte(*b, '}')
*b = trimSuffixWhitespace(*b)
if !hasSuffixByte(*b, ',') && !hasSuffixByte(*b, '{') {
*b = append(*b, ',')
}
} else {
err := errors.New("inlined raw value must be a JSON object")
return &SemanticError{action: "unmarshal", GoType: rawValueType, Err: err}
}
}
*b = append(*b, quotedName...)
*b = append(*b, ':')
rawValue, err := dec.ReadValue()
if err != nil {
return err
}
*b = append(*b, rawValue...)
*b = append(*b, '}')
return nil
} else {
name := string(unquotedName) // TODO: Intern this?
m := v
if m.IsNil() {
m.Set(reflect.MakeMap(m.Type()))
}
mk := reflect.ValueOf(name)
mv := newAddressableValue(v.Type().Elem()) // TODO: Cache across calls?
if v2 := m.MapIndex(mk); v2.IsValid() {
mv.Set(v2)
}
unmarshal := f.fncs.unmarshal
if uo.Unmarshalers != nil {
unmarshal, _ = uo.Unmarshalers.lookup(unmarshal, mv.Type())
}
err := unmarshal(uo, dec, mv)
m.SetMapIndex(mk, mv.Value)
if err != nil {
return err
}
return nil
}
}

View File

@ -0,0 +1,229 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"encoding"
"errors"
"reflect"
)
// Interfaces for custom serialization.
var (
jsonMarshalerV1Type = reflect.TypeOf((*MarshalerV1)(nil)).Elem()
jsonMarshalerV2Type = reflect.TypeOf((*MarshalerV2)(nil)).Elem()
jsonUnmarshalerV1Type = reflect.TypeOf((*UnmarshalerV1)(nil)).Elem()
jsonUnmarshalerV2Type = reflect.TypeOf((*UnmarshalerV2)(nil)).Elem()
textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
)
// MarshalerV1 is implemented by types that can marshal themselves.
// It is recommended that types implement MarshalerV2 unless
// the implementation is trying to avoid a hard dependency on this package.
//
// It is recommended that implementations return a buffer that is safe
// for the caller to retain and potentially mutate.
type MarshalerV1 interface {
MarshalJSON() ([]byte, error)
}
// MarshalerV2 is implemented by types that can marshal themselves.
// It is recommended that types implement MarshalerV2 instead of MarshalerV1
// since this is both more performant and flexible.
// If a type implements both MarshalerV1 and MarshalerV2,
// then MarshalerV2 takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default marshal options.
//
// The implementation must write only one JSON value to the Encoder and
// must not retain the pointer to Encoder.
type MarshalerV2 interface {
MarshalNextJSON(MarshalOptions, *Encoder) error
// TODO: Should users call the MarshalOptions.MarshalNext method or
// should/can they call this method directly? Does it matter?
}
// UnmarshalerV1 is implemented by types that can unmarshal themselves.
// It is recommended that types implement UnmarshalerV2 unless
// the implementation is trying to avoid a hard dependency on this package.
//
// The input can be assumed to be a valid encoding of a JSON value
// if called from unmarshal functionality in this package.
// UnmarshalJSON must copy the JSON data if it is retained after returning.
// It is recommended that UnmarshalJSON implement merge semantics when
// unmarshaling into a pre-populated value.
//
// Implementations must not retain or mutate the input []byte.
type UnmarshalerV1 interface {
UnmarshalJSON([]byte) error
}
// UnmarshalerV2 is implemented by types that can unmarshal themselves.
// It is recommended that types implement UnmarshalerV2 instead of UnmarshalerV1
// since this is both more performant and flexible.
// If a type implements both UnmarshalerV1 and UnmarshalerV2,
// then UnmarshalerV2 takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default unmarshal options.
//
// The implementation must read only one JSON value from the Decoder.
// It is recommended that UnmarshalNextJSON implement merge semantics when
// unmarshaling into a pre-populated value.
//
// Implementations must not retain the pointer to Decoder.
type UnmarshalerV2 interface {
UnmarshalNextJSON(UnmarshalOptions, *Decoder) error
// TODO: Should users call the UnmarshalOptions.UnmarshalNext method or
// should/can they call this method directly? Does it matter?
}
func makeMethodArshaler(fncs *arshaler, t reflect.Type) *arshaler {
// Avoid injecting method arshaler on the pointer or interface version
// to avoid ever calling the method on a nil pointer or interface receiver.
// Let it be injected on the value receiver (which is always addressable).
if t.Kind() == reflect.Pointer || t.Kind() == reflect.Interface {
return fncs
}
// Handle custom marshaler.
switch which, needAddr := implementsWhich(t, jsonMarshalerV2Type, jsonMarshalerV1Type, textMarshalerType); which {
case jsonMarshalerV2Type:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
prevDepth, prevLength := enc.tokens.depthLength()
err := va.addrWhen(needAddr).Interface().(MarshalerV2).MarshalNextJSON(mo, enc)
currDepth, currLength := enc.tokens.depthLength()
if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil {
err = errors.New("must write exactly one JSON value")
}
if err != nil {
err = wrapSkipFunc(err, "marshal method")
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
return nil
}
case jsonMarshalerV1Type:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
marshaler := va.addrWhen(needAddr).Interface().(MarshalerV1)
val, err := marshaler.MarshalJSON()
if err != nil {
err = wrapSkipFunc(err, "marshal method")
// TODO: Avoid wrapping semantic errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
if err := enc.WriteValue(val); err != nil {
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", JSONKind: RawValue(val).Kind(), GoType: t, Err: err}
}
return nil
}
case textMarshalerType:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
marshaler := va.addrWhen(needAddr).Interface().(encoding.TextMarshaler)
s, err := marshaler.MarshalText()
if err != nil {
err = wrapSkipFunc(err, "marshal method")
// TODO: Avoid wrapping semantic errors.
return &SemanticError{action: "marshal", JSONKind: '"', GoType: t, Err: err}
}
val := enc.UnusedBuffer()
val, err = appendString(val, string(s), true, nil)
if err != nil {
return &SemanticError{action: "marshal", JSONKind: '"', GoType: t, Err: err}
}
if err := enc.WriteValue(val); err != nil {
// TODO: Avoid wrapping syntactic or I/O errors.
return &SemanticError{action: "marshal", JSONKind: '"', GoType: t, Err: err}
}
return nil
}
}
// Handle custom unmarshaler.
switch which, needAddr := implementsWhich(t, jsonUnmarshalerV2Type, jsonUnmarshalerV1Type, textUnmarshalerType); which {
case jsonUnmarshalerV2Type:
fncs.nonDefault = true
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
prevDepth, prevLength := dec.tokens.depthLength()
err := va.addrWhen(needAddr).Interface().(UnmarshalerV2).UnmarshalNextJSON(uo, dec)
currDepth, currLength := dec.tokens.depthLength()
if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil {
err = errors.New("must read exactly one JSON value")
}
if err != nil {
err = wrapSkipFunc(err, "unmarshal method")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
return nil
}
case jsonUnmarshalerV1Type:
fncs.nonDefault = true
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
val, err := dec.ReadValue()
if err != nil {
return err // must be a syntactic or I/O error
}
unmarshaler := va.addrWhen(needAddr).Interface().(UnmarshalerV1)
if err := unmarshaler.UnmarshalJSON(val); err != nil {
err = wrapSkipFunc(err, "unmarshal method")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
return nil
}
case textUnmarshalerType:
fncs.nonDefault = true
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
var flags valueFlags
val, err := dec.readValue(&flags)
if err != nil {
return err // must be a syntactic or I/O error
}
if val.Kind() != '"' {
err = errors.New("JSON value must be string type")
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
s := unescapeStringMayCopy(val, flags.isVerbatim())
unmarshaler := va.addrWhen(needAddr).Interface().(encoding.TextUnmarshaler)
if err := unmarshaler.UnmarshalText(s); err != nil {
err = wrapSkipFunc(err, "unmarshal method")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
return nil
}
}
return fncs
}
// implementsWhich is like t.Implements(ifaceType) for a list of interfaces,
// but checks whether either t or reflect.PointerTo(t) implements the interface.
// It returns the first interface type that matches and whether a value of t
// needs to be addressed first before it implements the interface.
func implementsWhich(t reflect.Type, ifaceTypes ...reflect.Type) (which reflect.Type, needAddr bool) {
for _, ifaceType := range ifaceTypes {
switch {
case t.Implements(ifaceType):
return ifaceType, false
case reflect.PointerTo(t).Implements(ifaceType):
return ifaceType, true
}
}
return nil, false
}
// addrWhen returns va.Addr if addr is specified, otherwise it returns itself.
func (va addressableValue) addrWhen(addr bool) reflect.Value {
if addr {
return va.Addr()
}
return va.Value
}

View File

@ -0,0 +1,196 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"fmt"
"reflect"
"strings"
"time"
)
var (
timeDurationType = reflect.TypeOf((*time.Duration)(nil)).Elem()
timeTimeType = reflect.TypeOf((*time.Time)(nil)).Elem()
)
func makeTimeArshaler(fncs *arshaler, t reflect.Type) *arshaler {
// Ideally, time types would implement MarshalerV2 and UnmarshalerV2,
// but that would incur a dependency on package json from package time.
// Given how widely used time is, it is more acceptable that we incur a
// dependency on time from json.
//
// Injecting the arshaling functionality like this will not be identical
// to actually declaring methods on the time types since embedding of the
// time types will not be able to forward this functionality.
switch t {
case timeDurationType:
fncs.nonDefault = true
marshalNanos := fncs.marshal
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
if mo.format != "" && mo.formatDepth == enc.tokens.depth() {
if mo.format == "nanos" {
mo.format = ""
return marshalNanos(mo, enc, va)
} else {
return newInvalidFormatError("marshal", t, mo.format)
}
}
td := va.Interface().(time.Duration)
b := enc.UnusedBuffer()
b = append(b, '"')
b = append(b, td.String()...) // never contains special characters
b = append(b, '"')
return enc.WriteValue(b)
}
unmarshalNanos := fncs.unmarshal
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
// TODO: Should there be a flag that specifies that we can unmarshal
// from either form since there would be no ambiguity?
if uo.format != "" && uo.formatDepth == dec.tokens.depth() {
if uo.format == "nanos" {
uo.format = ""
return unmarshalNanos(uo, dec, va)
} else {
return newInvalidFormatError("unmarshal", t, uo.format)
}
}
var flags valueFlags
td := va.Addr().Interface().(*time.Duration)
val, err := dec.readValue(&flags)
if err != nil {
return err
}
switch k := val.Kind(); k {
case 'n':
*td = time.Duration(0)
return nil
case '"':
val = unescapeStringMayCopy(val, flags.isVerbatim())
td2, err := time.ParseDuration(string(val))
if err != nil {
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
}
*td = td2
return nil
default:
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t}
}
}
case timeTimeType:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
format := time.RFC3339Nano
if mo.format != "" && mo.formatDepth == enc.tokens.depth() {
var err error
format, err = checkTimeFormat(mo.format)
if err != nil {
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
}
tt := va.Interface().(time.Time)
if y := tt.Year(); y < 0 || y >= 10000 {
// RFC 3339 is clear that years are 4 digits exactly.
// See https://go.dev/issue/4556#c15 for more discussion.
err := fmt.Errorf("year %d outside of range [0,9999]", y)
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
b := enc.UnusedBuffer()
b = append(b, '"')
b = tt.AppendFormat(b, format)
b = append(b, '"')
// The format may contain special characters that need escaping.
// Verify that the result is a valid JSON string (common case),
// otherwise escape the string correctly (slower case).
if consumeSimpleString(b) != len(b) {
b, _ = appendString(nil, string(b[len(`"`):len(b)-len(`"`)]), true, nil)
}
return enc.WriteValue(b)
}
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
format := time.RFC3339Nano
if uo.format != "" && uo.formatDepth == dec.tokens.depth() {
var err error
format, err = checkTimeFormat(uo.format)
if err != nil {
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
}
var flags valueFlags
tt := va.Addr().Interface().(*time.Time)
val, err := dec.readValue(&flags)
if err != nil {
return err
}
k := val.Kind()
switch k {
case 'n':
*tt = time.Time{}
return nil
case '"':
val = unescapeStringMayCopy(val, flags.isVerbatim())
tt2, err := time.Parse(format, string(val))
if err != nil {
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
}
*tt = tt2
return nil
default:
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t}
}
}
}
return fncs
}
func checkTimeFormat(format string) (string, error) {
// We assume that an exported constant in the time package will
// always start with an uppercase ASCII letter.
if len(format) > 0 && 'A' <= format[0] && format[0] <= 'Z' {
switch format {
case "ANSIC":
return time.ANSIC, nil
case "UnixDate":
return time.UnixDate, nil
case "RubyDate":
return time.RubyDate, nil
case "RFC822":
return time.RFC822, nil
case "RFC822Z":
return time.RFC822Z, nil
case "RFC850":
return time.RFC850, nil
case "RFC1123":
return time.RFC1123, nil
case "RFC1123Z":
return time.RFC1123Z, nil
case "RFC3339":
return time.RFC3339, nil
case "RFC3339Nano":
return time.RFC3339Nano, nil
case "Kitchen":
return time.Kitchen, nil
case "Stamp":
return time.Stamp, nil
case "StampMilli":
return time.StampMilli, nil
case "StampMicro":
return time.StampMicro, nil
case "StampNano":
return time.StampNano, nil
default:
// Reject any format that is an exported Go identifier in case
// new format constants are added to the time package.
if strings.TrimFunc(format, isLetterOrDigit) == "" {
return "", fmt.Errorf("undefined format layout: %v", format)
}
}
}
return format, nil
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,185 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package json implements serialization of JSON
// as specified in RFC 4627, RFC 7159, RFC 7493, RFC 8259, and RFC 8785.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
//
// Terminology
//
// This package uses the terms "encode" and "decode" for syntactic functionality
// that is concerned with processing JSON based on its grammar, and
// uses the terms "marshal" and "unmarshal" for semantic functionality
// that determines the meaning of JSON values as Go values and vice-versa.
// It aims to provide a clear distinction between functionality that
// is purely concerned with encoding versus that of marshaling.
// For example, one can directly encode a stream of JSON tokens without
// needing to marshal a concrete Go value representing them.
// Similarly, one can decode a stream of JSON tokens without
// needing to unmarshal them into a concrete Go value.
//
// This package uses JSON terminology when discussing JSON, which may differ
// from related concepts in Go or elsewhere in computing literature.
//
// - A JSON "object" refers to an unordered collection of name/value members.
// - A JSON "array" refers to an ordered sequence of elements.
// - A JSON "value" refers to either a literal (i.e., null, false, or true),
// string, number, object, or array.
//
// See RFC 8259 for more information.
//
//
// Specifications
//
// Relevant specifications include RFC 4627, RFC 7159, RFC 7493, RFC 8259,
// and RFC 8785. Each RFC is generally a stricter subset of another RFC.
// In increasing order of strictness:
//
// - RFC 4627 and RFC 7159 do not require (but recommend) the use of UTF-8
// and also do not require (but recommend) that object names be unique.
// - RFC 8259 requires the use of UTF-8,
// but does not require (but recommends) that object names be unique.
// - RFC 7493 requires the use of UTF-8
// and also requires that object names be unique.
// - RFC 8785 defines a canonical representation. It requires the use of UTF-8
// and also requires that object names be unique and in a specific ordering.
// It specifies exactly how strings and numbers must be formatted.
//
// The primary difference between RFC 4627 and RFC 7159 is that the former
// restricted top-level values to only JSON objects and arrays, while
// RFC 7159 and subsequent RFCs permit top-level values to additionally be
// JSON nulls, booleans, strings, or numbers.
//
// By default, this package operates on RFC 7493, but can be configured
// to operate according to the other RFC specifications.
// RFC 7493 is a stricter subset of RFC 8259 and fully compliant with it.
// In particular, it makes specific choices about behavior that RFC 8259
// leaves as undefined in order to ensure greater interoperability.
//
//
// JSON Representation of Go structs
//
// A Go struct is naturally represented as a JSON object,
// where each Go struct field corresponds with a JSON object member.
// When marshaling, all Go struct fields are recursively encoded in depth-first
// order as JSON object members except those that are ignored or omitted.
// When unmarshaling, JSON object members are recursively decoded
// into the corresponding Go struct fields.
// Object members that do not match any struct fields,
// also known as “unknown members”, are ignored by default or rejected
// if UnmarshalOptions.RejectUnknownMembers is specified.
//
// The representation of each struct field can be customized in the
// "json" struct field tag, where the tag is a comma separated list of options.
// As a special case, if the entire tag is `json:"-"`,
// then the field is ignored with regard to its JSON representation.
//
// The first option is the JSON object name override for the Go struct field.
// If the name is not specified, then the Go struct field name
// is used as the JSON object name. JSON names containing commas or quotes,
// or names identical to "" or "-", can be specified using
// a single-quoted string literal, where the syntax is identical to
// the Go grammar for a double-quoted string literal,
// but instead uses single quotes as the delimiters.
// By default, unmarshaling uses case-sensitive matching to identify
// the Go struct field associated with a JSON object name.
//
// After the name, the following tag options are supported:
//
// - omitzero: When marshaling, the "omitzero" option specifies that
// the struct field should be omitted if the field value is zero
// as determined by the "IsZero() bool" method if present,
// otherwise based on whether the field is the zero Go value.
// This option has no effect when unmarshaling.
//
// - omitempty: When marshaling, the "omitempty" option specifies that
// the struct field should be omitted if the field value would have been
// encoded as a JSON null, empty string, empty object, or empty array.
// This option has no effect when unmarshaling.
//
// - string: The "string" option specifies that
// MarshalOptions.StringifyNumbers and UnmarshalOptions.StringifyNumbers
// be set when marshaling or unmarshaling a struct field value.
// This causes numeric types to be encoded as a JSON number
// within a JSON string, and to be decoded from either a JSON number or
// a JSON string containing a JSON number.
// This extra level of encoding is often necessary since
// many JSON parsers cannot precisely represent 64-bit integers.
//
// - nocase: When unmarshaling, the "nocase" option specifies that
// if the JSON object name does not exactly match the JSON name
// for any of the struct fields, then it attempts to match the struct field
// using a case-insensitive match that also ignores dashes and underscores.
// If multiple fields match, the first declared field in breadth-first order
// takes precedence. This option has no effect when marshaling.
//
// - inline: The "inline" option specifies that
// the JSON representable content of this field type is to be promoted
// as if they were specified in the parent struct.
// It is the JSON equivalent of Go struct embedding.
// A Go embedded field is implicitly inlined unless an explicit JSON name
// is specified. The inlined field must be a Go struct
// (that does not implement any JSON methods), RawValue, map[string]T,
// or an unnamed pointer to such types. When marshaling,
// inlined fields from a pointer type are omitted if it is nil.
// Inlined fields of type RawValue and map[string]T are called
// “inlined fallbacks” as they can represent all possible
// JSON object members not directly handled by the parent struct.
// Only one inlined fallback field may be specified in a struct,
// while many non-fallback fields may be specified. This option
// must not be specified with any other option (including the JSON name).
//
// - unknown: The "unknown" option is a specialized variant
// of the inlined fallback to indicate that this Go struct field
// contains any number of unknown JSON object members. The field type
// must be a RawValue, map[string]T, or an unnamed pointer to such types.
// If MarshalOptions.DiscardUnknownMembers is specified when marshaling,
// the contents of this field are ignored.
// If UnmarshalOptions.RejectUnknownMembers is specified when unmarshaling,
// any unknown object members are rejected regardless of whether
// an inlined fallback with the "unknown" option exists. This option
// must not be specified with any other option (including the JSON name).
//
// - format: The "format" option specifies a format flag
// used to specialize the formatting of the field value.
// The option is a key-value pair specified as "format:value" where
// the value must be either a literal consisting of letters and numbers
// (e.g., "format:RFC3339") or a single-quoted string literal
// (e.g., "format:'2006-01-02'"). The interpretation of the format flag
// is determined by the struct field type.
//
// The "omitzero" and "omitempty" options are mostly semantically identical.
// The former is defined in terms of the Go type system,
// while the latter in terms of the JSON type system.
// Consequently they behave differently in some circumstances.
// For example, only a nil slice or map is omitted under "omitzero", while
// an empty slice or map is omitted under "omitempty" regardless of nilness.
// The "omitzero" option is useful for types with a well-defined zero value
// (e.g., netip.Addr) or have an IsZero method (e.g., time.Time).
//
// Every Go struct corresponds to a list of JSON representable fields
// which is constructed by performing a breadth-first search over
// all struct fields (excluding unexported or ignored fields),
// where the search recursively descends into inlined structs.
// The set of non-inlined fields in a struct must have unique JSON names.
// If multiple fields all have the same JSON name, then the one
// at shallowest depth takes precedence and the other fields at deeper depths
// are excluded from the list of JSON representable fields.
// If multiple fields at the shallowest depth have the same JSON name,
// then all of those fields are excluded from the list. This is analogous to
// Go visibility rules for struct field selection with embedded struct types.
//
// Marshaling or unmarshaling a non-empty struct
// without any JSON representable fields results in a SemanticError.
// Unexported fields must not have any `json` tags except for `json:"-"`.
package json
// requireKeyedLiterals can be embedded in a struct to require keyed literals.
type requireKeyedLiterals struct{}
// nonComparable can be embedded in a struct to prevent comparability.
type nonComparable [0]func()

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,183 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"reflect"
"strconv"
"strings"
"unicode/utf8"
)
const errorPrefix = "json: "
// Error matches errors returned by this package according to errors.Is.
const Error = jsonError("json error")
type jsonError string
func (e jsonError) Error() string {
return string(e)
}
func (e jsonError) Is(target error) bool {
return e == target || target == Error
}
type ioError struct {
action string // either "read" or "write"
err error
}
func (e *ioError) Error() string {
return errorPrefix + e.action + " error: " + e.err.Error()
}
func (e *ioError) Unwrap() error {
return e.err
}
func (e *ioError) Is(target error) bool {
return e == target || target == Error || errors.Is(e.err, target)
}
// SemanticError describes an error determining the meaning
// of JSON data as Go data or vice-versa.
//
// The contents of this error as produced by this package may change over time.
type SemanticError struct {
requireKeyedLiterals
nonComparable
action string // either "marshal" or "unmarshal"
// ByteOffset indicates that an error occurred after this byte offset.
ByteOffset int64
// JSONPointer indicates that an error occurred within this JSON value
// as indicated using the JSON Pointer notation (see RFC 6901).
JSONPointer string
// JSONKind is the JSON kind that could not be handled.
JSONKind Kind // may be zero if unknown
// GoType is the Go type that could not be handled.
GoType reflect.Type // may be nil if unknown
// Err is the underlying error.
Err error // may be nil
}
func (e *SemanticError) Error() string {
var sb strings.Builder
sb.WriteString(errorPrefix)
// Hyrum-proof the error message by deliberately switching between
// two equivalent renderings of the same error message.
// The randomization is tied to the Hyrum-proofing already applied
// on map iteration in Go.
for phrase := range map[string]struct{}{"cannot": {}, "unable to": {}} {
sb.WriteString(phrase)
break // use whichever phrase we get in the first iteration
}
// Format action.
var preposition string
switch e.action {
case "marshal":
sb.WriteString(" marshal")
preposition = " from"
case "unmarshal":
sb.WriteString(" unmarshal")
preposition = " into"
default:
sb.WriteString(" handle")
preposition = " with"
}
// Format JSON kind.
var omitPreposition bool
switch e.JSONKind {
case 'n':
sb.WriteString(" JSON null")
case 'f', 't':
sb.WriteString(" JSON boolean")
case '"':
sb.WriteString(" JSON string")
case '0':
sb.WriteString(" JSON number")
case '{', '}':
sb.WriteString(" JSON object")
case '[', ']':
sb.WriteString(" JSON array")
default:
omitPreposition = true
}
// Format Go type.
if e.GoType != nil {
if !omitPreposition {
sb.WriteString(preposition)
}
sb.WriteString(" Go value of type ")
sb.WriteString(e.GoType.String())
}
// Format where.
switch {
case e.JSONPointer != "":
sb.WriteString(" within JSON value at ")
sb.WriteString(strconv.Quote(e.JSONPointer))
case e.ByteOffset > 0:
sb.WriteString(" after byte offset ")
sb.WriteString(strconv.FormatInt(e.ByteOffset, 10))
}
// Format underlying error.
if e.Err != nil {
sb.WriteString(": ")
sb.WriteString(e.Err.Error())
}
return sb.String()
}
func (e *SemanticError) Is(target error) bool {
return e == target || target == Error || errors.Is(e.Err, target)
}
func (e *SemanticError) Unwrap() error {
return e.Err
}
// SyntacticError is a description of a syntactic error that occurred when
// encoding or decoding JSON according to the grammar.
//
// The contents of this error as produced by this package may change over time.
type SyntacticError struct {
requireKeyedLiterals
nonComparable
// ByteOffset indicates that an error occurred after this byte offset.
ByteOffset int64
str string
}
func (e *SyntacticError) Error() string {
return errorPrefix + e.str
}
func (e *SyntacticError) Is(target error) bool {
return e == target || target == Error
}
func (e *SyntacticError) withOffset(pos int64) error {
return &SyntacticError{ByteOffset: pos, str: e.str}
}
func newInvalidCharacterError(prefix []byte, where string) *SyntacticError {
what := quoteRune(prefix)
return &SyntacticError{str: "invalid character " + what + " " + where}
}
func quoteRune(b []byte) string {
r, n := utf8.DecodeRune(b)
if r == utf8.RuneError && n == 1 {
return `'\x` + strconv.FormatUint(uint64(b[0]), 16) + `'`
}
return strconv.QuoteRune(r)
}

View File

@ -0,0 +1,509 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"fmt"
"io"
"reflect"
"sort"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
var errIgnoredField = errors.New("ignored field")
type isZeroer interface {
IsZero() bool
}
var isZeroerType = reflect.TypeOf((*isZeroer)(nil)).Elem()
type structFields struct {
flattened []structField // listed in depth-first ordering
byActualName map[string]*structField
byFoldedName map[string][]*structField
inlinedFallback *structField
}
type structField struct {
id int // unique numeric ID in breadth-first ordering
index []int // index into a struct according to reflect.Type.FieldByIndex
typ reflect.Type
fncs *arshaler
isZero func(addressableValue) bool
isEmpty func(addressableValue) bool
fieldOptions
}
func makeStructFields(root reflect.Type) (structFields, *SemanticError) {
var fs structFields
fs.byActualName = make(map[string]*structField, root.NumField())
fs.byFoldedName = make(map[string][]*structField, root.NumField())
// ambiguous is a sentinel value to indicate that at least two fields
// at the same depth have the same name, and thus cancel each other out.
// This follows the same rules as selecting a field on embedded structs
// where the shallowest field takes precedence. If more than one field
// exists at the shallowest depth, then the selection is illegal.
// See https://go.dev/ref/spec#Selectors.
ambiguous := new(structField)
// Setup a queue for a breath-first search.
var queueIndex int
type queueEntry struct {
typ reflect.Type
index []int
visitChildren bool // whether to recursively visit inlined field in this struct
}
queue := []queueEntry{{root, nil, true}}
seen := map[reflect.Type]bool{root: true}
// Perform a breadth-first search over all reachable fields.
// This ensures that len(f.index) will be monotonically increasing.
for queueIndex < len(queue) {
qe := queue[queueIndex]
queueIndex++
t := qe.typ
inlinedFallbackIndex := -1 // index of last inlined fallback field in current struct
namesIndex := make(map[string]int) // index of each field with a given JSON object name in current struct
var hasAnyJSONTag bool // whether any Go struct field has a `json` tag
var hasAnyJSONField bool // whether any JSON serializable fields exist in current struct
for i := 0; i < t.NumField(); i++ {
sf := t.Field(i)
_, hasTag := sf.Tag.Lookup("json")
hasAnyJSONTag = hasAnyJSONTag || hasTag
options, err := parseFieldOptions(sf)
if err != nil {
if err == errIgnoredField {
continue
}
return structFields{}, &SemanticError{GoType: t, Err: err}
}
hasAnyJSONField = true
f := structField{
// Allocate a new slice (len=N+1) to hold both
// the parent index (len=N) and the current index (len=1).
// Do this to avoid clobbering the memory of the parent index.
index: append(append(make([]int, 0, len(qe.index)+1), qe.index...), i),
typ: sf.Type,
fieldOptions: options,
}
if sf.Anonymous && !f.hasName {
f.inline = true // implied by use of Go embedding without an explicit name
}
if f.inline || f.unknown {
// Handle an inlined field that serializes to/from
// zero or more JSON object members.
if f.inline && f.unknown {
err := fmt.Errorf("Go struct field %s cannot have both `inline` and `unknown` specified", sf.Name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
switch f.fieldOptions {
case fieldOptions{name: f.name, quotedName: f.quotedName, inline: true}:
case fieldOptions{name: f.name, quotedName: f.quotedName, unknown: true}:
default:
err := fmt.Errorf("Go struct field %s cannot have any options other than `inline` or `unknown` specified", sf.Name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Unwrap one level of pointer indirection similar to how Go
// only allows embedding either T or *T, but not **T.
tf := f.typ
if tf.Kind() == reflect.Pointer && tf.Name() == "" {
tf = tf.Elem()
}
// Reject any types with custom serialization otherwise
// it becomes impossible to know what sub-fields to inline.
if which, _ := implementsWhich(tf,
jsonMarshalerV2Type, jsonMarshalerV1Type, textMarshalerType,
jsonUnmarshalerV2Type, jsonUnmarshalerV1Type, textUnmarshalerType,
); which != nil && tf != rawValueType {
err := fmt.Errorf("inlined Go struct field %s of type %s must not implement JSON marshal or unmarshal methods", sf.Name, tf)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Handle an inlined field that serializes to/from
// a finite number of JSON object members backed by a Go struct.
if tf.Kind() == reflect.Struct {
if f.unknown {
err := fmt.Errorf("inlined Go struct field %s of type %s with `unknown` tag must be a Go map of string key or a json.RawValue", sf.Name, tf)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
if qe.visitChildren {
queue = append(queue, queueEntry{tf, f.index, !seen[tf]})
}
seen[tf] = true
continue
}
// Handle an inlined field that serializes to/from any number of
// JSON object members back by a Go map or RawValue.
switch {
case tf == rawValueType:
f.fncs = nil // specially handled in arshal_inlined.go
case tf.Kind() == reflect.Map && tf.Key() == stringType:
f.fncs = lookupArshaler(tf.Elem())
default:
err := fmt.Errorf("inlined Go struct field %s of type %s must be a Go struct, Go map of string key, or json.RawValue", sf.Name, tf)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Reject multiple inlined fallback fields within the same struct.
if inlinedFallbackIndex >= 0 {
err := fmt.Errorf("inlined Go struct fields %s and %s cannot both be a Go map or json.RawValue", t.Field(inlinedFallbackIndex).Name, sf.Name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
inlinedFallbackIndex = i
// Multiple inlined fallback fields across different structs
// follow the same precedence rules as Go struct embedding.
if fs.inlinedFallback == nil {
fs.inlinedFallback = &f // store first occurrence at lowest depth
} else if len(fs.inlinedFallback.index) == len(f.index) {
fs.inlinedFallback = ambiguous // at least two occurrences at same depth
}
} else {
// Handle normal Go struct field that serializes to/from
// a single JSON object member.
// Provide a function that uses a type's IsZero method.
switch {
case sf.Type.Kind() == reflect.Interface && sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool {
// Avoid panics calling IsZero on a nil interface or
// non-nil interface with nil pointer.
return va.IsNil() || (va.Elem().Kind() == reflect.Pointer && va.Elem().IsNil()) || va.Interface().(isZeroer).IsZero()
}
case sf.Type.Kind() == reflect.Pointer && sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool {
// Avoid panics calling IsZero on nil pointer.
return va.IsNil() || va.Interface().(isZeroer).IsZero()
}
case sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool { return va.Interface().(isZeroer).IsZero() }
case reflect.PointerTo(sf.Type).Implements(isZeroerType):
f.isZero = func(va addressableValue) bool { return va.Addr().Interface().(isZeroer).IsZero() }
}
// Provide a function that can determine whether the value would
// serialize as an empty JSON value.
switch sf.Type.Kind() {
case reflect.String, reflect.Map, reflect.Array, reflect.Slice:
f.isEmpty = func(va addressableValue) bool { return va.Len() == 0 }
case reflect.Pointer, reflect.Interface:
f.isEmpty = func(va addressableValue) bool { return va.IsNil() }
}
f.id = len(fs.flattened)
f.fncs = lookupArshaler(sf.Type)
fs.flattened = append(fs.flattened, f)
// Reject user-specified names with invalid UTF-8.
if !utf8.ValidString(f.name) {
err := fmt.Errorf("Go struct field %s has JSON object name %q with invalid UTF-8", sf.Name, f.name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Reject multiple fields with same name within the same struct.
if j, ok := namesIndex[f.name]; ok {
err := fmt.Errorf("Go struct fields %s and %s conflict over JSON object name %q", t.Field(j).Name, sf.Name, f.name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
namesIndex[f.name] = i
// Multiple fields of the same name across different structs
// follow the same precedence rules as Go struct embedding.
if f2 := fs.byActualName[f.name]; f2 == nil {
fs.byActualName[f.name] = &fs.flattened[len(fs.flattened)-1] // store first occurrence at lowest depth
} else if len(f2.index) == len(f.index) {
fs.byActualName[f.name] = ambiguous // at least two occurrences at same depth
}
}
}
// NOTE: New users to the json package are occasionally surprised that
// unexported fields are ignored. This occurs by necessity due to our
// inability to directly introspect such fields with Go reflection
// without the use of unsafe.
//
// To reduce friction here, refuse to serialize any Go struct that
// has no JSON serializable fields, has at least one Go struct field,
// and does not have any `json` tags present. For example,
// errors returned by errors.New would fail to serialize.
isEmptyStruct := t.NumField() == 0
if !isEmptyStruct && !hasAnyJSONTag && !hasAnyJSONField {
err := errors.New("Go struct has no exported fields")
return structFields{}, &SemanticError{GoType: t, Err: err}
}
}
// Remove all fields that are duplicates.
// This may move elements forward to fill the holes from removed fields.
var n int
for _, f := range fs.flattened {
switch f2 := fs.byActualName[f.name]; {
case f2 == ambiguous:
delete(fs.byActualName, f.name)
case f2 == nil:
continue // may be nil due to previous delete
// TODO(https://go.dev/issue/45955): Use slices.Equal.
case reflect.DeepEqual(f.index, f2.index):
f.id = n
fs.flattened[n] = f
fs.byActualName[f.name] = &fs.flattened[n] // fix pointer to new location
n++
}
}
fs.flattened = fs.flattened[:n]
if fs.inlinedFallback == ambiguous {
fs.inlinedFallback = nil
}
if len(fs.flattened) != len(fs.byActualName) {
panic(fmt.Sprintf("BUG: flattened list of fields mismatches fields mapped by name: %d != %d", len(fs.flattened), len(fs.byActualName)))
}
// Sort the fields according to a depth-first ordering.
// This operation will cause pointers in byActualName to become incorrect,
// which we will correct in another loop shortly thereafter.
sort.Slice(fs.flattened, func(i, j int) bool {
si := fs.flattened[i].index
sj := fs.flattened[j].index
for len(si) > 0 && len(sj) > 0 {
switch {
case si[0] < sj[0]:
return true
case si[0] > sj[0]:
return false
default:
si = si[1:]
sj = sj[1:]
}
}
return len(si) < len(sj)
})
// Recompute the mapping of fields in the byActualName map.
// Pre-fold all names so that we can lookup folded names quickly.
for i, f := range fs.flattened {
foldedName := string(foldName([]byte(f.name)))
fs.byActualName[f.name] = &fs.flattened[i]
fs.byFoldedName[foldedName] = append(fs.byFoldedName[foldedName], &fs.flattened[i])
}
for foldedName, fields := range fs.byFoldedName {
if len(fields) > 1 {
// The precedence order for conflicting nocase names
// is by breadth-first order, rather than depth-first order.
sort.Slice(fields, func(i, j int) bool {
return fields[i].id < fields[j].id
})
fs.byFoldedName[foldedName] = fields
}
}
return fs, nil
}
type fieldOptions struct {
name string
quotedName string // quoted name per RFC 8785, section 3.2.2.2.
hasName bool
nocase bool
inline bool
unknown bool
omitzero bool
omitempty bool
string bool
format string
}
// parseFieldOptions parses the `json` tag in a Go struct field as
// a structured set of options configuring parameters such as
// the JSON member name and other features.
// As a special case, it returns errIgnoredField if the field is ignored.
func parseFieldOptions(sf reflect.StructField) (out fieldOptions, err error) {
tag, hasTag := sf.Tag.Lookup("json")
// Check whether this field is explicitly ignored.
if tag == "-" {
return fieldOptions{}, errIgnoredField
}
// Check whether this field is unexported.
if !sf.IsExported() {
// In contrast to v1, v2 no longer forwards exported fields from
// embedded fields of unexported types since Go reflection does not
// allow the same set of operations that are available in normal cases
// of purely exported fields.
// See https://go.dev/issue/21357 and https://go.dev/issue/24153.
if sf.Anonymous {
return fieldOptions{}, fmt.Errorf("embedded Go struct field %s of an unexported type must be explicitly ignored with a `json:\"-\"` tag", sf.Type.Name())
}
// Tag options specified on an unexported field suggests user error.
if hasTag {
return fieldOptions{}, fmt.Errorf("unexported Go struct field %s cannot have non-ignored `json:%q` tag", sf.Name, tag)
}
return fieldOptions{}, errIgnoredField
}
// Determine the JSON member name for this Go field. A user-specified name
// may be provided as either an identifier or a single-quoted string.
// The single-quoted string allows arbitrary characters in the name.
// See https://go.dev/issue/2718 and https://go.dev/issue/3546.
out.name = sf.Name // always starts with an uppercase character
if len(tag) > 0 && !strings.HasPrefix(tag, ",") {
// For better compatibility with v1, accept almost any unescaped name.
n := len(tag) - len(strings.TrimLeftFunc(tag, func(r rune) bool {
return !strings.ContainsRune(",\\'\"`", r) // reserve comma, backslash, and quotes
}))
opt := tag[:n]
if n == 0 {
// Allow a single quoted string for arbitrary names.
opt, n, err = consumeTagOption(tag)
if err != nil {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err)
}
}
out.hasName = true
out.name = opt
tag = tag[n:]
}
b, _ := appendString(nil, out.name, false, nil)
out.quotedName = string(b)
// Handle any additional tag options (if any).
var wasFormat bool
seenOpts := make(map[string]bool)
for len(tag) > 0 {
// Consume comma delimiter.
if tag[0] != ',' {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid character %q before next option (expecting ',')", sf.Name, tag[0])
}
tag = tag[len(","):]
if len(tag) == 0 {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid trailing ',' character", sf.Name)
}
// Consume and process the tag option.
opt, n, err := consumeTagOption(tag)
if err != nil {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err)
}
rawOpt := tag[:n]
tag = tag[n:]
switch {
case wasFormat:
return fieldOptions{}, fmt.Errorf("Go struct field %s has `format` tag option that was not specified last", sf.Name)
case strings.HasPrefix(rawOpt, "'") && strings.TrimFunc(opt, isLetterOrDigit) == "":
return fieldOptions{}, fmt.Errorf("Go struct field %s has unnecessarily quoted appearance of `%s` tag option; specify `%s` instead", sf.Name, rawOpt, opt)
}
switch opt {
case "nocase":
out.nocase = true
case "inline":
out.inline = true
case "unknown":
out.unknown = true
case "omitzero":
out.omitzero = true
case "omitempty":
out.omitempty = true
case "string":
out.string = true
case "format":
if !strings.HasPrefix(tag, ":") {
return fieldOptions{}, fmt.Errorf("Go struct field %s is missing value for `format` tag option", sf.Name)
}
tag = tag[len(":"):]
opt, n, err := consumeTagOption(tag)
if err != nil {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed value for `format` tag option: %v", sf.Name, err)
}
tag = tag[n:]
out.format = opt
wasFormat = true
default:
// Reject keys that resemble one of the supported options.
// This catches invalid mutants such as "omitEmpty" or "omit_empty".
normOpt := strings.ReplaceAll(strings.ToLower(opt), "_", "")
switch normOpt {
case "nocase", "inline", "unknown", "omitzero", "omitempty", "string", "format":
return fieldOptions{}, fmt.Errorf("Go struct field %s has invalid appearance of `%s` tag option; specify `%s` instead", sf.Name, opt, normOpt)
}
// NOTE: Everything else is ignored. This does not mean it is
// forward compatible to insert arbitrary tag options since
// a future version of this package may understand that tag.
}
// Reject duplicates.
if seenOpts[opt] {
return fieldOptions{}, fmt.Errorf("Go struct field %s has duplicate appearance of `%s` tag option", sf.Name, rawOpt)
}
seenOpts[opt] = true
}
return out, nil
}
func consumeTagOption(in string) (string, int, error) {
switch r, _ := utf8.DecodeRuneInString(in); {
// Option as a Go identifier.
case r == '_' || unicode.IsLetter(r):
n := len(in) - len(strings.TrimLeftFunc(in, isLetterOrDigit))
return in[:n], n, nil
// Option as a single-quoted string.
case r == '\'':
// The grammar is nearly identical to a double-quoted Go string literal,
// but uses single quotes as the terminators. The reason for a custom
// grammar is because both backtick and double quotes cannot be used
// verbatim in a struct tag.
//
// Convert a single-quoted string to a double-quote string and rely on
// strconv.Unquote to handle the rest.
var inEscape bool
b := []byte{'"'}
n := len(`'`)
for len(in) > n {
r, rn := utf8.DecodeRuneInString(in[n:])
switch {
case inEscape:
if r == '\'' {
b = b[:len(b)-1] // remove escape character: `\'` => `'`
}
inEscape = false
case r == '\\':
inEscape = true
case r == '"':
b = append(b, '\\') // insert escape character: `"` => `\"`
case r == '\'':
b = append(b, '"')
n += len(`'`)
out, err := strconv.Unquote(string(b))
if err != nil {
return "", 0, fmt.Errorf("invalid single-quoted string: %s", in[:n])
}
return out, n, nil
}
b = append(b, in[n:][:rn]...)
n += rn
}
if n > 10 {
n = 10 // limit the amount of context printed in the error
}
return "", 0, fmt.Errorf("single-quoted string not terminated: %s...", in[:n])
case len(in) == 0:
return "", 0, io.ErrUnexpectedEOF
default:
return "", 0, fmt.Errorf("invalid character %q at start of option (expecting Unicode letter or single quote)", r)
}
}
func isLetterOrDigit(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
}

View File

@ -0,0 +1,56 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"unicode"
"unicode/utf8"
)
// foldName returns a folded string such that foldName(x) == foldName(y)
// is similar to strings.EqualFold(x, y), but ignores underscore and dashes.
// This allows foldName to match common naming conventions.
func foldName(in []byte) []byte {
// This is inlinable to take advantage of "function outlining".
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
var arr [32]byte // large enough for most JSON names
return appendFoldedName(arr[:0], in)
}
func appendFoldedName(out, in []byte) []byte {
for i := 0; i < len(in); {
// Handle single-byte ASCII.
if c := in[i]; c < utf8.RuneSelf {
if c != '_' && c != '-' {
if 'a' <= c && c <= 'z' {
c -= 'a' - 'A'
}
out = append(out, c)
}
i++
continue
}
// Handle multi-byte Unicode.
r, n := utf8.DecodeRune(in[i:])
out = utf8.AppendRune(out, foldRune(r))
i += n
}
return out
}
// foldRune is a variation on unicode.SimpleFold that returns the same rune
// for all runes in the same fold set.
//
// Invariant:
//
// foldRune(x) == foldRune(y) ⇔ strings.EqualFold(string(x), string(y))
func foldRune(r rune) rune {
for {
r2 := unicode.SimpleFold(r)
if r2 <= r {
return r2 // smallest character in the fold set
}
r = r2
}
}

View File

@ -0,0 +1,86 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"encoding/binary"
"math/bits"
)
// stringCache is a cache for strings converted from a []byte.
type stringCache [256]string // 256*unsafe.Sizeof(string("")) => 4KiB
// make returns the string form of b.
// It returns a pre-allocated string from c if present, otherwise
// it allocates a new string, inserts it into the cache, and returns it.
func (c *stringCache) make(b []byte) string {
const (
minCachedLen = 2 // single byte strings are already interned by the runtime
maxCachedLen = 256 // large enough for UUIDs, IPv6 addresses, SHA-256 checksums, etc.
)
if c == nil || len(b) < minCachedLen || len(b) > maxCachedLen {
return string(b)
}
// Compute a hash from the fixed-width prefix and suffix of the string.
// This ensures hashing a string is a constant time operation.
var h uint32
switch {
case len(b) >= 8:
lo := binary.LittleEndian.Uint64(b[:8])
hi := binary.LittleEndian.Uint64(b[len(b)-8:])
h = hash64(uint32(lo), uint32(lo>>32)) ^ hash64(uint32(hi), uint32(hi>>32))
case len(b) >= 4:
lo := binary.LittleEndian.Uint32(b[:4])
hi := binary.LittleEndian.Uint32(b[len(b)-4:])
h = hash64(lo, hi)
case len(b) >= 2:
lo := binary.LittleEndian.Uint16(b[:2])
hi := binary.LittleEndian.Uint16(b[len(b)-2:])
h = hash64(uint32(lo), uint32(hi))
}
// Check the cache for the string.
i := h % uint32(len(*c))
if s := (*c)[i]; s == string(b) {
return s
}
s := string(b)
(*c)[i] = s
return s
}
// hash64 returns the hash of two uint32s as a single uint32.
func hash64(lo, hi uint32) uint32 {
// If avalanche=true, this is identical to XXH32 hash on a 8B string:
// var b [8]byte
// binary.LittleEndian.PutUint32(b[:4], lo)
// binary.LittleEndian.PutUint32(b[4:], hi)
// return xxhash.Sum32(b[:])
const (
prime1 = 0x9e3779b1
prime2 = 0x85ebca77
prime3 = 0xc2b2ae3d
prime4 = 0x27d4eb2f
prime5 = 0x165667b1
)
h := prime5 + uint32(8)
h += lo * prime3
h = bits.RotateLeft32(h, 17) * prime4
h += hi * prime3
h = bits.RotateLeft32(h, 17) * prime4
// Skip final mix (avalanche) step of XXH32 for performance reasons.
// Empirical testing shows that the improvements in unbiased distribution
// does not outweigh the extra cost in computational complexity.
const avalanche = false
if avalanche {
h ^= h >> 15
h *= prime2
h ^= h >> 13
h *= prime3
h ^= h >> 16
}
return h
}

View File

@ -0,0 +1,150 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"bytes"
"io"
"math/bits"
"sync"
)
// TODO(https://go.dev/issue/47657): Use sync.PoolOf.
var (
// This owns the internal buffer since there is no io.Writer to output to.
// Since the buffer can get arbitrarily large in normal usage,
// there is statistical tracking logic to determine whether to recycle
// the internal buffer or not based on a history of utilization.
bufferedEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON before flushing it to the underlying io.Writer.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
)
// bufferStatistics is statistics to track buffer utilization.
// It is used to determine whether to recycle a buffer or not
// to avoid https://go.dev/issue/23199.
type bufferStatistics struct {
strikes int // number of times the buffer was under-utilized
prevLen int // length of previous buffer
}
func getBufferedEncoder(o EncodeOptions) *Encoder {
e := bufferedEncoderPool.Get().(*Encoder)
if e.buf == nil {
// Round up to nearest 2ⁿ to make best use of malloc size classes.
// See runtime/sizeclasses.go on Go1.15.
// Logical OR with 63 to ensure 64 as the minimum buffer size.
n := 1 << bits.Len(uint(e.bufStats.prevLen|63))
e.buf = make([]byte, 0, n)
}
e.reset(e.buf[:0], nil, o)
return e
}
func putBufferedEncoder(e *Encoder) {
// Recycle large buffers only if sufficiently utilized.
// If a buffer is under-utilized enough times sequentially,
// then it is discarded, ensuring that a single large buffer
// won't be kept alive by a continuous stream of small usages.
//
// The worst case utilization is computed as:
// MIN_UTILIZATION_THRESHOLD / (1 + MAX_NUM_STRIKES)
//
// For the constants chosen below, this is (25%)/(1+4) ⇒ 5%.
// This may seem low, but it ensures a lower bound on
// the absolute worst-case utilization. Without this check,
// this would be theoretically 0%, which is infinitely worse.
//
// See https://go.dev/issue/27735.
switch {
case cap(e.buf) <= 4<<10: // always recycle buffers smaller than 4KiB
e.bufStats.strikes = 0
case cap(e.buf)/4 <= len(e.buf): // at least 25% utilization
e.bufStats.strikes = 0
case e.bufStats.strikes < 4: // at most 4 strikes
e.bufStats.strikes++
default: // discard the buffer; too large and too often under-utilized
e.bufStats.strikes = 0
e.bufStats.prevLen = len(e.buf) // heuristic for size to allocate next time
e.buf = nil
}
bufferedEncoderPool.Put(e)
}
func getStreamingEncoder(w io.Writer, o EncodeOptions) *Encoder {
if _, ok := w.(*bytes.Buffer); ok {
e := bytesBufferEncoderPool.Get().(*Encoder)
e.reset(nil, w, o) // buffer taken from bytes.Buffer
return e
} else {
e := streamingEncoderPool.Get().(*Encoder)
e.reset(e.buf[:0], w, o) // preserve existing buffer
return e
}
}
func putStreamingEncoder(e *Encoder) {
if _, ok := e.wr.(*bytes.Buffer); ok {
bytesBufferEncoderPool.Put(e)
} else {
if cap(e.buf) > 64<<10 {
e.buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingEncoderPool.Put(e)
}
}
var (
// This does not own the internal buffer since it is externally provided.
bufferedDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON fetched from the underlying io.Reader.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferDecoderPool = bufferedDecoderPool
)
func getBufferedDecoder(b []byte, o DecodeOptions) *Decoder {
d := bufferedDecoderPool.Get().(*Decoder)
d.reset(b, nil, o)
return d
}
func putBufferedDecoder(d *Decoder) {
bufferedDecoderPool.Put(d)
}
func getStreamingDecoder(r io.Reader, o DecodeOptions) *Decoder {
if _, ok := r.(*bytes.Buffer); ok {
d := bytesBufferDecoderPool.Get().(*Decoder)
d.reset(nil, r, o) // buffer taken from bytes.Buffer
return d
} else {
d := streamingDecoderPool.Get().(*Decoder)
d.reset(d.buf[:0], r, o) // preserve existing buffer
return d
}
}
func putStreamingDecoder(d *Decoder) {
if _, ok := d.rd.(*bytes.Buffer); ok {
bytesBufferDecoderPool.Put(d)
} else {
if cap(d.buf) > 64<<10 {
d.buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingDecoderPool.Put(d)
}
}

View File

@ -0,0 +1,747 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"math"
"strconv"
)
var (
errMissingName = &SyntacticError{str: "missing string for object name"}
errMissingColon = &SyntacticError{str: "missing character ':' after object name"}
errMissingValue = &SyntacticError{str: "missing value after object name"}
errMissingComma = &SyntacticError{str: "missing character ',' after object or array value"}
errMismatchDelim = &SyntacticError{str: "mismatching structural token for object or array"}
)
const errInvalidNamespace = jsonError("object namespace is in an invalid state")
type state struct {
// tokens validates whether the next token kind is valid.
tokens stateMachine
// names is a stack of object names.
// Not used if AllowDuplicateNames is true.
names objectNameStack
// namespaces is a stack of object namespaces.
// For performance reasons, Encoder or Decoder may not update this
// if Marshal or Unmarshal is able to track names in a more efficient way.
// See makeMapArshaler and makeStructArshaler.
// Not used if AllowDuplicateNames is true.
namespaces objectNamespaceStack
}
func (s *state) reset() {
s.tokens.reset()
s.names.reset()
s.namespaces.reset()
}
// appendStackPointer appends a JSON Pointer (RFC 6901) to the current value.
// The returned pointer is only accurate if s.names is populated,
// otherwise it uses the numeric index as the object member name.
//
// Invariant: Must call s.names.copyQuotedBuffer beforehand.
func (s state) appendStackPointer(b []byte) []byte {
var objectDepth int
for i := 1; i < s.tokens.depth(); i++ {
e := s.tokens.index(i)
if e.length() == 0 {
break // empty object or array
}
b = append(b, '/')
switch {
case e.isObject():
if objectDepth < s.names.length() {
for _, c := range s.names.getUnquoted(objectDepth) {
// Per RFC 6901, section 3, escape '~' and '/' characters.
switch c {
case '~':
b = append(b, "~0"...)
case '/':
b = append(b, "~1"...)
default:
b = append(b, c)
}
}
} else {
// Since the names stack is unpopulated, the name is unknown.
// As a best-effort replacement, use the numeric member index.
// While inaccurate, it produces a syntactically valid pointer.
b = strconv.AppendUint(b, uint64((e.length()-1)/2), 10)
}
objectDepth++
case e.isArray():
b = strconv.AppendUint(b, uint64(e.length()-1), 10)
}
}
return b
}
// stateMachine is a push-down automaton that validates whether
// a sequence of tokens is valid or not according to the JSON grammar.
// It is useful for both encoding and decoding.
//
// It is a stack where each entry represents a nested JSON object or array.
// The stack has a minimum depth of 1 where the first level is a
// virtual JSON array to handle a stream of top-level JSON values.
// The top-level virtual JSON array is special in that it doesn't require commas
// between each JSON value.
//
// For performance, most methods are carefully written to be inlineable.
// The zero value is a valid state machine ready for use.
type stateMachine struct {
stack []stateEntry
last stateEntry
}
// reset resets the state machine.
// The machine always starts with a minimum depth of 1.
func (m *stateMachine) reset() {
m.stack = m.stack[:0]
if cap(m.stack) > 1<<10 {
m.stack = nil
}
m.last = stateTypeArray
}
// depth is the current nested depth of JSON objects and arrays.
// It is one-indexed (i.e., top-level values have a depth of 1).
func (m stateMachine) depth() int {
return len(m.stack) + 1
}
// index returns a reference to the ith entry.
// It is only valid until the next push method call.
func (m *stateMachine) index(i int) *stateEntry {
if i == len(m.stack) {
return &m.last
}
return &m.stack[i]
}
// depthLength reports the current nested depth and
// the length of the last JSON object or array.
func (m stateMachine) depthLength() (int, int) {
return m.depth(), m.last.length()
}
// appendLiteral appends a JSON literal as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendLiteral() error {
switch {
case m.last.needObjectName():
return errMissingName
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
return nil
}
}
// appendString appends a JSON string as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendString() error {
switch {
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
return nil
}
}
// appendNumber appends a JSON number as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendNumber() error {
return m.appendLiteral()
}
// pushObject appends a JSON start object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushObject() error {
switch {
case m.last.needObjectName():
return errMissingName
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
m.stack = append(m.stack, m.last)
m.last = stateTypeObject
return nil
}
}
// popObject appends a JSON end object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popObject() error {
switch {
case !m.last.isObject():
return errMismatchDelim
case m.last.needObjectValue():
return errMissingValue
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last = m.stack[len(m.stack)-1]
m.stack = m.stack[:len(m.stack)-1]
return nil
}
}
// pushArray appends a JSON start array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushArray() error {
switch {
case m.last.needObjectName():
return errMissingName
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
m.stack = append(m.stack, m.last)
m.last = stateTypeArray
return nil
}
}
// popArray appends a JSON end array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popArray() error {
switch {
case !m.last.isArray() || len(m.stack) == 0: // forbid popping top-level virtual JSON array
return errMismatchDelim
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last = m.stack[len(m.stack)-1]
m.stack = m.stack[:len(m.stack)-1]
return nil
}
}
// needIndent reports whether indent whitespace should be injected.
// A zero value means that no whitespace should be injected.
// A positive value means '\n', indentPrefix, and (n-1) copies of indentBody
// should be appended to the output immediately before the next token.
func (m stateMachine) needIndent(next Kind) (n int) {
willEnd := next == '}' || next == ']'
switch {
case m.depth() == 1:
return 0 // top-level values are never indented
case m.last.length() == 0 && willEnd:
return 0 // an empty object or array is never indented
case m.last.length() == 0 || m.last.needImplicitComma(next):
return m.depth()
case willEnd:
return m.depth() - 1
default:
return 0
}
}
// mayAppendDelim appends a colon or comma that may precede the next token.
func (m stateMachine) mayAppendDelim(b []byte, next Kind) []byte {
switch {
case m.last.needImplicitColon():
return append(b, ':')
case m.last.needImplicitComma(next) && len(m.stack) != 0: // comma not needed for top-level values
return append(b, ',')
default:
return b
}
}
// needDelim reports whether a colon or comma token should be implicitly emitted
// before the next token of the specified kind.
// A zero value means no delimiter should be emitted.
func (m stateMachine) needDelim(next Kind) (delim byte) {
switch {
case m.last.needImplicitColon():
return ':'
case m.last.needImplicitComma(next) && len(m.stack) != 0: // comma not needed for top-level values
return ','
default:
return 0
}
}
// checkDelim reports whether the specified delimiter should be there given
// the kind of the next token that appears immediately afterwards.
func (m stateMachine) checkDelim(delim byte, next Kind) error {
switch needDelim := m.needDelim(next); {
case needDelim == delim:
return nil
case needDelim == ':':
return errMissingColon
case needDelim == ',':
return errMissingComma
default:
return newInvalidCharacterError([]byte{delim}, "before next token")
}
}
// invalidateDisabledNamespaces marks all disabled namespaces as invalid.
//
// For efficiency, Marshal and Unmarshal may disable namespaces since there are
// more efficient ways to track duplicate names. However, if an error occurs,
// the namespaces in Encoder or Decoder will be left in an inconsistent state.
// Mark the namespaces as invalid so that future method calls on
// Encoder or Decoder will return an error.
func (m *stateMachine) invalidateDisabledNamespaces() {
for i := 0; i < m.depth(); i++ {
e := m.index(i)
if !e.isActiveNamespace() {
e.invalidateNamespace()
}
}
}
// stateEntry encodes several artifacts within a single unsigned integer:
// - whether this represents a JSON object or array,
// - whether this object should check for duplicate names, and
// - how many elements are in this JSON object or array.
type stateEntry uint64
const (
// The type mask (1 bit) records whether this is a JSON object or array.
stateTypeMask stateEntry = 0x8000_0000_0000_0000
stateTypeObject stateEntry = 0x8000_0000_0000_0000
stateTypeArray stateEntry = 0x0000_0000_0000_0000
// The name check mask (2 bit) records whether to update
// the namespaces for the current JSON object and
// whether the namespace is valid.
stateNamespaceMask stateEntry = 0x6000_0000_0000_0000
stateDisableNamespace stateEntry = 0x4000_0000_0000_0000
stateInvalidNamespace stateEntry = 0x2000_0000_0000_0000
// The count mask (61 bits) records the number of elements.
stateCountMask stateEntry = 0x1fff_ffff_ffff_ffff
stateCountLSBMask stateEntry = 0x0000_0000_0000_0001
stateCountOdd stateEntry = 0x0000_0000_0000_0001
stateCountEven stateEntry = 0x0000_0000_0000_0000
)
// length reports the number of elements in the JSON object or array.
// Each name and value in an object entry is treated as a separate element.
func (e stateEntry) length() int {
return int(e & stateCountMask)
}
// isObject reports whether this is a JSON object.
func (e stateEntry) isObject() bool {
return e&stateTypeMask == stateTypeObject
}
// isArray reports whether this is a JSON array.
func (e stateEntry) isArray() bool {
return e&stateTypeMask == stateTypeArray
}
// needObjectName reports whether the next token must be a JSON string,
// which is necessary for JSON object names.
func (e stateEntry) needObjectName() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountEven
}
// needImplicitColon reports whether an colon should occur next,
// which always occurs after JSON object names.
func (e stateEntry) needImplicitColon() bool {
return e.needObjectValue()
}
// needObjectValue reports whether the next token must be a JSON value,
// which is necessary after every JSON object name.
func (e stateEntry) needObjectValue() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountOdd
}
// needImplicitComma reports whether an comma should occur next,
// which always occurs after a value in a JSON object or array
// before the next value (or name).
func (e stateEntry) needImplicitComma(next Kind) bool {
return !e.needObjectValue() && e.length() > 0 && next != '}' && next != ']'
}
// increment increments the number of elements for the current object or array.
// This assumes that overflow won't practically be an issue since
// 1<<bits.OnesCount(stateCountMask) is sufficiently large.
func (e *stateEntry) increment() {
(*e)++
}
// decrement decrements the number of elements for the current object or array.
// It is the callers responsibility to ensure that e.length > 0.
func (e *stateEntry) decrement() {
(*e)--
}
// disableNamespace disables the JSON object namespace such that the
// Encoder or Decoder no longer updates the namespace.
func (e *stateEntry) disableNamespace() {
*e |= stateDisableNamespace
}
// isActiveNamespace reports whether the JSON object namespace is actively
// being updated and used for duplicate name checks.
func (e stateEntry) isActiveNamespace() bool {
return e&(stateDisableNamespace) == 0
}
// invalidateNamespace marks the JSON object namespace as being invalid.
func (e *stateEntry) invalidateNamespace() {
*e |= stateInvalidNamespace
}
// isValidNamespace reports whether the JSON object namespace is valid.
func (e stateEntry) isValidNamespace() bool {
return e&(stateInvalidNamespace) == 0
}
// objectNameStack is a stack of names when descending into a JSON object.
// In contrast to objectNamespaceStack, this only has to remember a single name
// per JSON object.
//
// This data structure may contain offsets to encodeBuffer or decodeBuffer.
// It violates clean abstraction of layers, but is significantly more efficient.
// This ensures that popping and pushing in the common case is a trivial
// push/pop of an offset integer.
//
// The zero value is an empty names stack ready for use.
type objectNameStack struct {
// offsets is a stack of offsets for each name.
// A non-negative offset is the ending offset into the local names buffer.
// A negative offset is the bit-wise inverse of a starting offset into
// a remote buffer (e.g., encodeBuffer or decodeBuffer).
// A math.MinInt offset at the end implies that the last object is empty.
// Invariant: Positive offsets always occur before negative offsets.
offsets []int
// unquotedNames is a back-to-back concatenation of names.
unquotedNames []byte
}
func (ns *objectNameStack) reset() {
ns.offsets = ns.offsets[:0]
ns.unquotedNames = ns.unquotedNames[:0]
if cap(ns.offsets) > 1<<6 {
ns.offsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.unquotedNames) > 1<<10 {
ns.unquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
func (ns *objectNameStack) length() int {
return len(ns.offsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
// It returns an empty string if the last object is empty.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) getUnquoted(i int) []byte {
ns.ensureCopiedBuffer()
if i == 0 {
return ns.unquotedNames[:ns.offsets[0]]
} else {
return ns.unquotedNames[ns.offsets[i-1]:ns.offsets[i-0]]
}
}
// invalidOffset indicates that the last JSON object currently has no name.
const invalidOffset = math.MinInt
// push descends into a nested JSON object.
func (ns *objectNameStack) push() {
ns.offsets = append(ns.offsets, invalidOffset)
}
// replaceLastQuotedOffset replaces the last name with the starting offset
// to the quoted name in some remote buffer. All offsets provided must be
// relative to the same buffer until copyQuotedBuffer is called.
func (ns *objectNameStack) replaceLastQuotedOffset(i int) {
// Use bit-wise inversion instead of naive multiplication by -1 to avoid
// ambiguity regarding zero (which is a valid offset into the names field).
// Bit-wise inversion is mathematically equivalent to -i-1,
// such that 0 becomes -1, 1 becomes -2, and so forth.
// This ensures that remote offsets are always negative.
ns.offsets[len(ns.offsets)-1] = ^i
}
// replaceLastUnquotedName replaces the last name with the provided name.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) replaceLastUnquotedName(s string) {
ns.ensureCopiedBuffer()
var startOffset int
if len(ns.offsets) > 1 {
startOffset = ns.offsets[len(ns.offsets)-2]
}
ns.unquotedNames = append(ns.unquotedNames[:startOffset], s...)
ns.offsets[len(ns.offsets)-1] = len(ns.unquotedNames)
}
// clearLast removes any name in the last JSON object.
// It is semantically equivalent to ns.push followed by ns.pop.
func (ns *objectNameStack) clearLast() {
ns.offsets[len(ns.offsets)-1] = invalidOffset
}
// pop ascends out of a nested JSON object.
func (ns *objectNameStack) pop() {
ns.offsets = ns.offsets[:len(ns.offsets)-1]
}
// copyQuotedBuffer copies names from the remote buffer into the local names
// buffer so that there are no more offset references into the remote buffer.
// This allows the remote buffer to change contents without affecting
// the names that this data structure is trying to remember.
func (ns *objectNameStack) copyQuotedBuffer(b []byte) {
// Find the first negative offset.
var i int
for i = len(ns.offsets) - 1; i >= 0 && ns.offsets[i] < 0; i-- {
continue
}
// Copy each name from the remote buffer into the local buffer.
for i = i + 1; i < len(ns.offsets); i++ {
if i == len(ns.offsets)-1 && ns.offsets[i] == invalidOffset {
if i == 0 {
ns.offsets[i] = 0
} else {
ns.offsets[i] = ns.offsets[i-1]
}
break // last JSON object had a push without any names
}
// As a form of Hyrum proofing, we write an invalid character into the
// buffer to make misuse of Decoder.ReadToken more obvious.
// We need to undo that mutation here.
quotedName := b[^ns.offsets[i]:]
if quotedName[0] == invalidateBufferByte {
quotedName[0] = '"'
}
// Append the unquoted name to the local buffer.
var startOffset int
if i > 0 {
startOffset = ns.offsets[i-1]
}
if n := consumeSimpleString(quotedName); n > 0 {
ns.unquotedNames = append(ns.unquotedNames[:startOffset], quotedName[len(`"`):n-len(`"`)]...)
} else {
ns.unquotedNames, _ = unescapeString(ns.unquotedNames[:startOffset], quotedName)
}
ns.offsets[i] = len(ns.unquotedNames)
}
}
func (ns *objectNameStack) ensureCopiedBuffer() {
if len(ns.offsets) > 0 && ns.offsets[len(ns.offsets)-1] < 0 {
panic("BUG: copyQuotedBuffer not called beforehand")
}
}
// objectNamespaceStack is a stack of object namespaces.
// This data structure assists in detecting duplicate names.
type objectNamespaceStack []objectNamespace
// reset resets the object namespace stack.
func (nss *objectNamespaceStack) reset() {
if cap(*nss) > 1<<10 {
*nss = nil
}
*nss = (*nss)[:0]
}
// push starts a new namespace for a nested JSON object.
func (nss *objectNamespaceStack) push() {
if cap(*nss) > len(*nss) {
*nss = (*nss)[:len(*nss)+1]
nss.last().reset()
} else {
*nss = append(*nss, objectNamespace{})
}
}
// last returns a pointer to the last JSON object namespace.
func (nss objectNamespaceStack) last() *objectNamespace {
return &nss[len(nss)-1]
}
// pop terminates the namespace for a nested JSON object.
func (nss *objectNamespaceStack) pop() {
*nss = (*nss)[:len(*nss)-1]
}
// objectNamespace is the namespace for a JSON object.
// In contrast to objectNameStack, this needs to remember a all names
// per JSON object.
//
// The zero value is an empty namespace ready for use.
type objectNamespace struct {
// It relies on a linear search over all the names before switching
// to use a Go map for direct lookup.
// endOffsets is a list of offsets to the end of each name in buffers.
// The length of offsets is the number of names in the namespace.
endOffsets []uint
// allUnquotedNames is a back-to-back concatenation of every name in the namespace.
allUnquotedNames []byte
// mapNames is a Go map containing every name in the namespace.
// Only valid if non-nil.
mapNames map[string]struct{}
}
// reset resets the namespace to be empty.
func (ns *objectNamespace) reset() {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
ns.mapNames = nil
if cap(ns.endOffsets) > 1<<6 {
ns.endOffsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.allUnquotedNames) > 1<<10 {
ns.allUnquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
// length reports the number of names in the namespace.
func (ns *objectNamespace) length() int {
return len(ns.endOffsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
func (ns *objectNamespace) getUnquoted(i int) []byte {
if i == 0 {
return ns.allUnquotedNames[:ns.endOffsets[0]]
} else {
return ns.allUnquotedNames[ns.endOffsets[i-1]:ns.endOffsets[i-0]]
}
}
// lastUnquoted retrieves the last name in the namespace.
func (ns *objectNamespace) lastUnquoted() []byte {
return ns.getUnquoted(ns.length() - 1)
}
// insertQuoted inserts a name and reports whether it was inserted,
// which only occurs if name is not already in the namespace.
// The provided name must be a valid JSON string.
func (ns *objectNamespace) insertQuoted(name []byte, isVerbatim bool) bool {
if isVerbatim {
name = name[len(`"`) : len(name)-len(`"`)]
}
return ns.insert(name, !isVerbatim)
}
func (ns *objectNamespace) insertUnquoted(name []byte) bool {
return ns.insert(name, false)
}
func (ns *objectNamespace) insert(name []byte, quoted bool) bool {
var allNames []byte
if quoted {
allNames, _ = unescapeString(ns.allUnquotedNames, name)
} else {
allNames = append(ns.allUnquotedNames, name...)
}
name = allNames[len(ns.allUnquotedNames):]
// Switch to a map if the buffer is too large for linear search.
// This does not add the current name to the map.
if ns.mapNames == nil && (ns.length() > 64 || len(ns.allUnquotedNames) > 1024) {
ns.mapNames = make(map[string]struct{})
var startOffset uint
for _, endOffset := range ns.endOffsets {
name := ns.allUnquotedNames[startOffset:endOffset]
ns.mapNames[string(name)] = struct{}{} // allocates a new string
startOffset = endOffset
}
}
if ns.mapNames == nil {
// Perform linear search over the buffer to find matching names.
// It provides O(n) lookup, but does not require any allocations.
var startOffset uint
for _, endOffset := range ns.endOffsets {
if string(ns.allUnquotedNames[startOffset:endOffset]) == string(name) {
return false
}
startOffset = endOffset
}
} else {
// Use the map if it is populated.
// It provides O(1) lookup, but requires a string allocation per name.
if _, ok := ns.mapNames[string(name)]; ok {
return false
}
ns.mapNames[string(name)] = struct{}{} // allocates a new string
}
ns.allUnquotedNames = allNames
ns.endOffsets = append(ns.endOffsets, uint(len(ns.allUnquotedNames)))
return true
}
// removeLast removes the last name in the namespace.
func (ns *objectNamespace) removeLast() {
if ns.mapNames != nil {
delete(ns.mapNames, string(ns.lastUnquoted()))
}
if ns.length()-1 == 0 {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
} else {
ns.endOffsets = ns.endOffsets[:ns.length()-1]
ns.allUnquotedNames = ns.allUnquotedNames[:ns.endOffsets[ns.length()-1]]
}
}
type uintSet64 uint64
func (s uintSet64) has(i uint) bool { return s&(1<<i) > 0 }
func (s *uintSet64) set(i uint) { *s |= 1 << i }
// uintSet is a set of unsigned integers.
// It is optimized for most integers being close to zero.
type uintSet struct {
lo uintSet64
hi []uintSet64
}
// has reports whether i is in the set.
func (s *uintSet) has(i uint) bool {
if i < 64 {
return s.lo.has(i)
} else {
i -= 64
iHi, iLo := int(i/64), uint(i%64)
return iHi < len(s.hi) && s.hi[iHi].has(iLo)
}
}
// insert inserts i into the set and reports whether it was the first insertion.
func (s *uintSet) insert(i uint) bool {
// TODO: Make this inlineable at least for the lower 64-bit case.
if i < 64 {
has := s.lo.has(i)
s.lo.set(i)
return !has
} else {
i -= 64
iHi, iLo := int(i/64), uint(i%64)
if iHi >= len(s.hi) {
s.hi = append(s.hi, make([]uintSet64, iHi+1-len(s.hi))...)
s.hi = s.hi[:cap(s.hi)]
}
has := s.hi[iHi].has(iLo)
s.hi[iHi].set(iLo)
return !has
}
}

View File

@ -0,0 +1,522 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"math"
"strconv"
)
// NOTE: Token is analogous to v1 json.Token.
const (
maxInt64 = math.MaxInt64
minInt64 = math.MinInt64
maxUint64 = math.MaxUint64
minUint64 = 0 // for consistency and readability purposes
invalidTokenPanic = "invalid json.Token; it has been voided by a subsequent json.Decoder call"
)
// Token represents a lexical JSON token, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a start or end delimiter for a JSON object (i.e., { or } )
// - a start or end delimiter for a JSON array (i.e., [ or ] )
//
// A Token cannot represent entire array or object values, while a RawValue can.
// There is no Token to represent commas and colons since
// these structural tokens can be inferred from the surrounding context.
type Token struct {
nonComparable
// Tokens can exist in either a "raw" or an "exact" form.
// Tokens produced by the Decoder are in the "raw" form.
// Tokens returned by constructors are usually in the "exact" form.
// The Encoder accepts Tokens in either the "raw" or "exact" form.
//
// The following chart shows the possible values for each Token type:
// ╔═════════════════╦════════════╤════════════╤════════════╗
// ║ Token type ║ raw field │ str field │ num field ║
// ╠═════════════════╬════════════╪════════════╪════════════╣
// ║ null (raw) ║ "null" │ "" │ 0 ║
// ║ false (raw) ║ "false" │ "" │ 0 ║
// ║ true (raw) ║ "true" │ "" │ 0 ║
// ║ string (raw) ║ non-empty │ "" │ offset ║
// ║ string (string) ║ nil │ non-empty │ 0 ║
// ║ number (raw) ║ non-empty │ "" │ offset ║
// ║ number (float) ║ nil │ "f" │ non-zero ║
// ║ number (int64) ║ nil │ "i" │ non-zero ║
// ║ number (uint64) ║ nil │ "u" │ non-zero ║
// ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
// ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
// ╚═════════════════╩════════════╧════════════╧════════════╝
//
// Notes:
// - For tokens stored in "raw" form, the num field contains the
// absolute offset determined by raw.previousOffsetStart().
// The buffer itself is stored in raw.previousBuffer().
// - JSON literals and structural characters are always in the "raw" form.
// - JSON strings and numbers can be in either "raw" or "exact" forms.
// - The exact zero value of JSON strings and numbers in the "exact" forms
// have ambiguous representation. Thus, they are always represented
// in the "raw" form.
// raw contains a reference to the raw decode buffer.
// If non-nil, then its value takes precedence over str and num.
// It is only valid if num == raw.previousOffsetStart().
raw *decodeBuffer
// str is the unescaped JSON string if num is zero.
// Otherwise, it is "f", "i", or "u" if num should be interpreted
// as a float64, int64, or uint64, respectively.
str string
// num is a float64, int64, or uint64 stored as a uint64 value.
// It is non-zero for any JSON number in the "exact" form.
num uint64
}
// TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
var (
Null Token = rawToken("null")
False Token = rawToken("false")
True Token = rawToken("true")
ObjectStart Token = rawToken("{")
ObjectEnd Token = rawToken("}")
ArrayStart Token = rawToken("[")
ArrayEnd Token = rawToken("]")
zeroString Token = rawToken(`""`)
zeroNumber Token = rawToken(`0`)
nanString Token = String("NaN")
pinfString Token = String("Infinity")
ninfString Token = String("-Infinity")
)
func rawToken(s string) Token {
return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
}
// Bool constructs a Token representing a JSON boolean.
func Bool(b bool) Token {
if b {
return True
}
return False
}
// String construct a Token representing a JSON string.
// The provided string should contain valid UTF-8, otherwise invalid characters
// may be mangled as the Unicode replacement character.
func String(s string) Token {
if len(s) == 0 {
return zeroString
}
return Token{str: s}
}
// Float constructs a Token representing a JSON number.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
func Float(n float64) Token {
switch {
case math.Float64bits(n) == 0:
return zeroNumber
case math.IsNaN(n):
return nanString
case math.IsInf(n, +1):
return pinfString
case math.IsInf(n, -1):
return ninfString
}
return Token{str: "f", num: math.Float64bits(n)}
}
// Int constructs a Token representing a JSON number from an int64.
func Int(n int64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "i", num: uint64(n)}
}
// Uint constructs a Token representing a JSON number from a uint64.
func Uint(n uint64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "u", num: uint64(n)}
}
// Clone makes a copy of the Token such that its value remains valid
// even after a subsequent Decoder.Read call.
func (t Token) Clone() Token {
// TODO: Allow caller to avoid any allocations?
if raw := t.raw; raw != nil {
// Avoid copying globals.
if t.raw.prevStart == 0 {
switch t.raw {
case Null.raw:
return Null
case False.raw:
return False
case True.raw:
return True
case ObjectStart.raw:
return ObjectStart
case ObjectEnd.raw:
return ObjectEnd
case ArrayStart.raw:
return ArrayStart
case ArrayEnd.raw:
return ArrayEnd
}
}
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
// TODO(https://go.dev/issue/45038): Use bytes.Clone.
buf := append([]byte(nil), raw.previousBuffer()...)
return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
}
return t
}
// Bool returns the value for a JSON boolean.
// It panics if the token kind is not a JSON boolean.
func (t Token) Bool() bool {
switch t.raw {
case True.raw:
return true
case False.raw:
return false
default:
panic("invalid JSON token kind: " + t.Kind().String())
}
}
// appendString appends a JSON string to dst and returns it.
// It panics if t is not a JSON string.
func (t Token) appendString(dst []byte, validateUTF8, preserveRaw bool, escapeRune func(rune) bool) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw string value.
buf := raw.previousBuffer()
if Kind(buf[0]) == '"' {
if escapeRune == nil && consumeSimpleString(buf) == len(buf) {
return append(dst, buf...), nil
}
dst, _, err := reformatString(dst, buf, validateUTF8, preserveRaw, escapeRune)
return dst, err
}
} else if len(t.str) != 0 && t.num == 0 {
// Handle exact string value.
return appendString(dst, t.str, validateUTF8, escapeRune)
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// String returns the unescaped string value for a JSON string.
// For other JSON kinds, this returns the raw JSON represention.
func (t Token) String() string {
// This is inlinable to take advantage of "function outlining".
// This avoids an allocation for the string(b) conversion
// if the caller does not use the string in an escaping manner.
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
s, b := t.string()
if len(b) > 0 {
return string(b)
}
return s
}
func (t Token) string() (string, []byte) {
if raw := t.raw; raw != nil {
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if buf[0] == '"' {
// TODO: Preserve valueFlags in Token?
isVerbatim := consumeSimpleString(buf) == len(buf)
return "", unescapeStringMayCopy(buf, isVerbatim)
}
// Handle tokens that are not JSON strings for fmt.Stringer.
return "", buf
}
if len(t.str) != 0 && t.num == 0 {
return t.str, nil
}
// Handle tokens that are not JSON strings for fmt.Stringer.
if t.num > 0 {
switch t.str[0] {
case 'f':
return string(appendNumber(nil, math.Float64frombits(t.num), 64)), nil
case 'i':
return strconv.FormatInt(int64(t.num), 10), nil
case 'u':
return strconv.FormatUint(uint64(t.num), 10), nil
}
}
return "<invalid json.Token>", nil
}
// appendNumber appends a JSON number to dst and returns it.
// It panics if t is not a JSON number.
func (t Token) appendNumber(dst []byte, canonicalize bool) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw number value.
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
if !canonicalize {
return append(dst, buf...), nil
}
dst, _, err := reformatNumber(dst, buf, canonicalize)
return dst, err
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'f':
return appendNumber(dst, math.Float64frombits(t.num), 64), nil
case 'i':
return strconv.AppendInt(dst, int64(t.num), 10), nil
case 'u':
return strconv.AppendUint(dst, uint64(t.num), 10), nil
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Float returns the floating-point value for a JSON number.
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
// It panics for all other cases.
func (t Token) Float() float64 {
if raw := t.raw; raw != nil {
// Handle raw number value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
fv, _ := parseFloat(buf, 64)
return fv
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'f':
return math.Float64frombits(t.num)
case 'i':
return float64(int64(t.num))
case 'u':
return float64(uint64(t.num))
}
}
// Handle string values with "NaN", "Infinity", or "-Infinity".
if t.Kind() == '"' {
switch t.String() {
case "NaN":
return math.NaN()
case "Infinity":
return math.Inf(+1)
case "-Infinity":
return math.Inf(-1)
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Int returns the signed integer value for a JSON number.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an int64 will be saturated
// to the closest representable value.
// It panics if the token kind is not a JSON number.
func (t Token) Int() int64 {
if raw := t.raw; raw != nil {
// Handle raw integer value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
neg := false
buf := raw.previousBuffer()
if len(buf) > 0 && buf[0] == '-' {
neg, buf = true, buf[1:]
}
if numAbs, ok := parseDecUint(buf); ok {
if neg {
if numAbs > -minInt64 {
return minInt64
}
return -1 * int64(numAbs)
} else {
if numAbs > +maxInt64 {
return maxInt64
}
return +1 * int64(numAbs)
}
}
} else if t.num != 0 {
// Handle exact integer value.
switch t.str[0] {
case 'i':
return int64(t.num)
case 'u':
if uint64(t.num) > maxInt64 {
return maxInt64
}
return int64(uint64(t.num))
}
}
// Handle JSON number that is a floating-point value.
if t.Kind() == '0' {
switch fv := t.Float(); {
case fv >= maxInt64:
return maxInt64
case fv <= minInt64:
return minInt64
default:
return int64(fv) // truncation toward zero
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Uint returns the unsigned integer value for a JSON number.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an uint64 will be saturated
// to the closest representable value.
// It panics if the token kind is not a JSON number.
func (t Token) Uint() uint64 {
// NOTE: This accessor returns 0 for any negative JSON number,
// which might be surprising, but is at least consistent with the behavior
// of saturating out-of-bounds numbers to the closest representable number.
if raw := t.raw; raw != nil {
// Handle raw integer value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
neg := false
buf := raw.previousBuffer()
if len(buf) > 0 && buf[0] == '-' {
neg, buf = true, buf[1:]
}
if num, ok := parseDecUint(buf); ok {
if neg {
return minUint64
}
return num
}
} else if t.num != 0 {
// Handle exact integer value.
switch t.str[0] {
case 'u':
return uint64(t.num)
case 'i':
if int64(t.num) < minUint64 {
return minUint64
}
return uint64(int64(t.num))
}
}
// Handle JSON number that is a floating-point value.
if t.Kind() == '0' {
switch fv := t.Float(); {
case fv >= maxUint64:
return maxUint64
case fv <= minUint64:
return minUint64
default:
return uint64(fv) // truncation toward zero
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Kind returns the token kind.
func (t Token) Kind() Kind {
switch {
case t.raw != nil:
raw := t.raw
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
return Kind(t.raw.buf[raw.prevStart]).normalize()
case t.num != 0:
return '0'
case len(t.str) != 0:
return '"'
default:
return invalidKind
}
}
// Kind represents each possible JSON token kind with a single byte,
// which is conveniently the first byte of that kind's grammar
// with the restriction that numbers always be represented with '0':
//
// - 'n': null
// - 'f': false
// - 't': true
// - '"': string
// - '0': number
// - '{': object start
// - '}': object end
// - '[': array start
// - ']': array end
//
// An invalid kind is usually represented using 0,
// but may be non-zero due to invalid JSON data.
type Kind byte
const invalidKind Kind = 0
// String prints the kind in a humanly readable fashion.
func (k Kind) String() string {
switch k {
case 'n':
return "null"
case 'f':
return "false"
case 't':
return "true"
case '"':
return "string"
case '0':
return "number"
case '{':
return "{"
case '}':
return "}"
case '[':
return "["
case ']':
return "]"
default:
return "<invalid json.Kind: " + quoteRune([]byte{byte(k)}) + ">"
}
}
// normalize coalesces all possible starting characters of a number as just '0'.
func (k Kind) normalize() Kind {
if k == '-' || ('0' <= k && k <= '9') {
return '0'
}
return k
}

View File

@ -0,0 +1,375 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"bytes"
"errors"
"io"
"sort"
"sync"
"unicode/utf16"
"unicode/utf8"
)
// NOTE: RawValue is analogous to v1 json.RawMessage.
// RawValue represents a single raw JSON value, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - an entire JSON object (e.g., {"fizz":"buzz"} )
// - an entire JSON array (e.g., [1,2,3] )
//
// RawValue can represent entire array or object values, while Token cannot.
// RawValue may contain leading and/or trailing whitespace.
type RawValue []byte
// Clone returns a copy of v.
func (v RawValue) Clone() RawValue {
if v == nil {
return nil
}
return append(RawValue{}, v...)
}
// String returns the string formatting of v.
func (v RawValue) String() string {
if v == nil {
return "null"
}
return string(v)
}
// IsValid reports whether the raw JSON value is syntactically valid
// according to RFC 7493.
//
// It verifies whether the input is properly encoded as UTF-8,
// that escape sequences within strings decode to valid Unicode codepoints, and
// that all names in each object are unique.
// It does not verify whether numbers are representable within the limits
// of any common numeric type (e.g., float64, int64, or uint64).
func (v RawValue) IsValid() bool {
d := getBufferedDecoder(v, DecodeOptions{})
defer putBufferedDecoder(d)
_, errVal := d.ReadValue()
_, errEOF := d.ReadToken()
return errVal == nil && errEOF == io.EOF
}
// Compact removes all whitespace from the raw JSON value.
//
// It does not reformat JSON strings to use any other representation.
// It is guaranteed to succeed if the input is valid.
// If the value is already compacted, then the buffer is not mutated.
func (v *RawValue) Compact() error {
return v.reformat(false, false, "", "")
}
// Indent reformats the whitespace in the raw JSON value so that each element
// in a JSON object or array begins on a new, indented line beginning with
// prefix followed by one or more copies of indent according to the nesting.
// The value does not begin with the prefix nor any indention,
// to make it easier to embed inside other formatted JSON data.
//
// It does not reformat JSON strings to use any other representation.
// It is guaranteed to succeed if the input is valid.
// If the value is already indented properly, then the buffer is not mutated.
func (v *RawValue) Indent(prefix, indent string) error {
return v.reformat(false, true, prefix, indent)
}
// Canonicalize canonicalizes the raw JSON value according to the
// JSON Canonicalization Scheme (JCS) as defined by RFC 8785
// where it produces a stable representation of a JSON value.
//
// The output stability is dependent on the stability of the application data
// (see RFC 8785, Appendix E). It cannot produce stable output from
// fundamentally unstable input. For example, if the JSON value
// contains ephemeral data (e.g., a frequently changing timestamp),
// then the value is still unstable regardless of whether this is called.
//
// Note that JCS treats all JSON numbers as IEEE 754 double precision numbers.
// Any numbers with precision beyond what is representable by that form
// will lose their precision when canonicalized. For example, integer values
// beyond ±2⁵³ will lose their precision. It is recommended that
// int64 and uint64 data types be represented as a JSON string.
//
// It is guaranteed to succeed if the input is valid.
// If the value is already canonicalized, then the buffer is not mutated.
func (v *RawValue) Canonicalize() error {
return v.reformat(true, false, "", "")
}
// TODO: Instead of implementing the v1 Marshaler/Unmarshaler,
// consider implementing the v2 versions instead.
// MarshalJSON returns v as the JSON encoding of v.
// It returns the stored value as the raw JSON output without any validation.
// If v is nil, then this returns a JSON null.
func (v RawValue) MarshalJSON() ([]byte, error) {
// NOTE: This matches the behavior of v1 json.RawMessage.MarshalJSON.
if v == nil {
return []byte("null"), nil
}
return v, nil
}
// UnmarshalJSON sets v as the JSON encoding of b.
// It stores a copy of the provided raw JSON input without any validation.
func (v *RawValue) UnmarshalJSON(b []byte) error {
// NOTE: This matches the behavior of v1 json.RawMessage.UnmarshalJSON.
if v == nil {
return errors.New("json.RawValue: UnmarshalJSON on nil pointer")
}
*v = append((*v)[:0], b...)
return nil
}
// Kind returns the starting token kind.
// For a valid value, this will never include '}' or ']'.
func (v RawValue) Kind() Kind {
if v := v[consumeWhitespace(v):]; len(v) > 0 {
return Kind(v[0]).normalize()
}
return invalidKind
}
func (v *RawValue) reformat(canonical, multiline bool, prefix, indent string) error {
var eo EncodeOptions
if canonical {
eo.AllowInvalidUTF8 = false // per RFC 8785, section 3.2.4
eo.AllowDuplicateNames = false // per RFC 8785, section 3.1
eo.canonicalizeNumbers = true // per RFC 8785, section 3.2.2.3
eo.EscapeRune = nil // per RFC 8785, section 3.2.2.2
eo.multiline = false // per RFC 8785, section 3.2.1
} else {
if s := trimLeftSpaceTab(prefix); len(s) > 0 {
panic("json: invalid character " + quoteRune([]byte(s)) + " in indent prefix")
}
if s := trimLeftSpaceTab(indent); len(s) > 0 {
panic("json: invalid character " + quoteRune([]byte(s)) + " in indent")
}
eo.AllowInvalidUTF8 = true
eo.AllowDuplicateNames = true
eo.preserveRawStrings = true
eo.multiline = multiline // in case indent is empty
eo.IndentPrefix = prefix
eo.Indent = indent
}
eo.omitTopLevelNewline = true
// Write the entire value to reformat all tokens and whitespace.
e := getBufferedEncoder(eo)
defer putBufferedEncoder(e)
if err := e.WriteValue(*v); err != nil {
return err
}
// For canonical output, we may need to reorder object members.
if canonical {
// Obtain a buffered encoder just to use its internal buffer as
// a scratch buffer in reorderObjects for reordering object members.
e2 := getBufferedEncoder(EncodeOptions{})
defer putBufferedEncoder(e2)
// Disable redundant checks performed earlier during encoding.
d := getBufferedDecoder(e.buf, DecodeOptions{AllowInvalidUTF8: true, AllowDuplicateNames: true})
defer putBufferedDecoder(d)
reorderObjects(d, &e2.buf) // per RFC 8785, section 3.2.3
}
// Store the result back into the value if different.
if !bytes.Equal(*v, e.buf) {
*v = append((*v)[:0], e.buf...)
}
return nil
}
func trimLeftSpaceTab(s string) string {
for i, r := range s {
switch r {
case ' ', '\t':
default:
return s[i:]
}
}
return ""
}
type memberName struct {
// name is the unescaped name.
name []byte
// before and after are byte offsets into Decoder.buf that represents
// the entire name/value pair. It may contain leading commas.
before, after int64
}
var memberNamePool = sync.Pool{New: func() any { return new(memberNames) }}
func getMemberNames() *memberNames {
ns := memberNamePool.Get().(*memberNames)
*ns = (*ns)[:0]
return ns
}
func putMemberNames(ns *memberNames) {
if cap(*ns) < 1<<10 {
for i := range *ns {
(*ns)[i] = memberName{} // avoid pinning name
}
memberNamePool.Put(ns)
}
}
type memberNames []memberName
func (m *memberNames) Len() int { return len(*m) }
func (m *memberNames) Less(i, j int) bool { return lessUTF16((*m)[i].name, (*m)[j].name) }
func (m *memberNames) Swap(i, j int) { (*m)[i], (*m)[j] = (*m)[j], (*m)[i] }
// reorderObjects recursively reorders all object members in place
// according to the ordering specified in RFC 8785, section 3.2.3.
//
// Pre-conditions:
// - The value is valid (i.e., no decoder errors should ever occur).
// - The value is compact (i.e., no whitespace is present).
// - Initial call is provided a Decoder reading from the start of v.
//
// Post-conditions:
// - Exactly one JSON value is read from the Decoder.
// - All fully-parsed JSON objects are reordered by directly moving
// the members in the value buffer.
//
// The runtime is approximately O(n·log(n)) + O(m·log(m)),
// where n is len(v) and m is the total number of object members.
func reorderObjects(d *Decoder, scratch *[]byte) {
switch tok, _ := d.ReadToken(); tok.Kind() {
case '{':
// Iterate and collect the name and offsets for every object member.
members := getMemberNames()
defer putMemberNames(members)
var prevName []byte
isSorted := true
beforeBody := d.InputOffset() // offset after '{'
for d.PeekKind() != '}' {
beforeName := d.InputOffset()
var flags valueFlags
name, _ := d.readValue(&flags)
name = unescapeStringMayCopy(name, flags.isVerbatim())
reorderObjects(d, scratch)
afterValue := d.InputOffset()
if isSorted && len(*members) > 0 {
isSorted = lessUTF16(prevName, name)
}
*members = append(*members, memberName{name, beforeName, afterValue})
prevName = name
}
afterBody := d.InputOffset() // offset before '}'
d.ReadToken()
// Sort the members; return early if it's already sorted.
if isSorted {
return
}
// TODO(https://go.dev/issue/47619): Use slices.Sort.
sort.Sort(members)
// Append the reordered members to a new buffer,
// then copy the reordered members back over the original members.
// Avoid swapping in place since each member may be a different size
// where moving a member over a smaller member may corrupt the data
// for subsequent members before they have been moved.
//
// The following invariant must hold:
// sum([m.after-m.before for m in members]) == afterBody-beforeBody
sorted := (*scratch)[:0]
for i, member := range *members {
if d.buf[member.before] == ',' {
member.before++ // trim leading comma
}
sorted = append(sorted, d.buf[member.before:member.after]...)
if i < len(*members)-1 {
sorted = append(sorted, ',') // append trailing comma
}
}
if int(afterBody-beforeBody) != len(sorted) {
panic("BUG: length invariant violated")
}
copy(d.buf[beforeBody:afterBody], sorted)
// Update scratch buffer to the largest amount ever used.
if len(sorted) > len(*scratch) {
*scratch = sorted
}
case '[':
for d.PeekKind() != ']' {
reorderObjects(d, scratch)
}
d.ReadToken()
}
}
// lessUTF16 reports whether x is lexicographically less than y according
// to the UTF-16 codepoints of the UTF-8 encoded input strings.
// This implements the ordering specified in RFC 8785, section 3.2.3.
// The inputs must be valid UTF-8, otherwise this may panic.
func lessUTF16(x, y []byte) bool {
// NOTE: This is an optimized, allocation-free implementation
// of lessUTF16Simple in fuzz_test.go. FuzzLessUTF16 verifies that the
// two implementations agree on the result of comparing any two strings.
isUTF16Self := func(r rune) bool {
return ('\u0000' <= r && r <= '\uD7FF') || ('\uE000' <= r && r <= '\uFFFF')
}
for {
if len(x) == 0 || len(y) == 0 {
return len(x) < len(y)
}
// ASCII fast-path.
if x[0] < utf8.RuneSelf || y[0] < utf8.RuneSelf {
if x[0] != y[0] {
return x[0] < y[0]
}
x, y = x[1:], y[1:]
continue
}
// Decode next pair of runes as UTF-8.
rx, nx := utf8.DecodeRune(x)
ry, ny := utf8.DecodeRune(y)
switch {
// Both runes encode as either a single or surrogate pair
// of UTF-16 codepoints.
case isUTF16Self(rx) == isUTF16Self(ry):
if rx != ry {
return rx < ry
}
// The x rune is a single UTF-16 codepoint, while
// the y rune is a surrogate pair of UTF-16 codepoints.
case isUTF16Self(rx):
ry, _ := utf16.EncodeRune(ry)
if rx != ry {
return rx < ry
}
panic("BUG: invalid UTF-8") // implies rx is an unpaired surrogate half
// The y rune is a single UTF-16 codepoint, while
// the x rune is a surrogate pair of UTF-16 codepoints.
case isUTF16Self(ry):
rx, _ := utf16.EncodeRune(rx)
if rx != ry {
return rx < ry
}
panic("BUG: invalid UTF-8") // implies ry is an unpaired surrogate half
}
x, y = x[nx:], y[ny:]
}
}

502
vendor/k8s.io/kube-openapi/pkg/validation/spec/fuzz.go generated vendored Normal file
View File

@ -0,0 +1,502 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"github.com/go-openapi/jsonreference"
"github.com/google/go-cmp/cmp"
fuzz "github.com/google/gofuzz"
)
var SwaggerFuzzFuncs []interface{} = []interface{}{
func(v *Responses, c fuzz.Continue) {
c.FuzzNoCustom(v)
if v.Default != nil {
// Check if we hit maxDepth and left an incomplete value
if v.Default.Description == "" {
v.Default = nil
v.StatusCodeResponses = nil
}
}
// conversion has no way to discern empty statusCodeResponses from
// nil, since "default" is always included in the map.
// So avoid empty responses list
if len(v.StatusCodeResponses) == 0 {
v.StatusCodeResponses = nil
}
},
func(v *Operation, c fuzz.Continue) {
c.FuzzNoCustom(v)
if v != nil {
// force non-nil
v.Responses = &Responses{}
c.Fuzz(v.Responses)
v.Schemes = nil
if c.RandBool() {
v.Schemes = append(v.Schemes, "http")
}
if c.RandBool() {
v.Schemes = append(v.Schemes, "https")
}
if c.RandBool() {
v.Schemes = append(v.Schemes, "ws")
}
if c.RandBool() {
v.Schemes = append(v.Schemes, "wss")
}
// Gnostic unconditionally makes security values non-null
// So do not fuzz null values into the array.
for i, val := range v.Security {
if val == nil {
v.Security[i] = make(map[string][]string)
}
for k, v := range val {
if v == nil {
val[k] = make([]string, 0)
}
}
}
}
},
func(v map[int]Response, c fuzz.Continue) {
n := 0
c.Fuzz(&n)
if n == 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
return
}
// Prevent negative numbers
num := c.Intn(4)
for i := 0; i < num+2; i++ {
val := Response{}
c.Fuzz(&val)
val.Description = c.RandString() + "x"
v[100*(i+1)+c.Intn(100)] = val
}
},
func(v map[string]PathItem, c fuzz.Continue) {
n := 0
c.Fuzz(&n)
if n == 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
return
}
num := c.Intn(5)
for i := 0; i < num+2; i++ {
val := PathItem{}
c.Fuzz(&val)
// Ref params are only allowed in certain locations, so
// possibly add a few to PathItems
numRefsToAdd := c.Intn(5)
for i := 0; i < numRefsToAdd; i++ {
theRef := Parameter{}
c.Fuzz(&theRef.Refable)
val.Parameters = append(val.Parameters, theRef)
}
v["/"+c.RandString()] = val
}
},
func(v *SchemaOrArray, c fuzz.Continue) {
*v = SchemaOrArray{}
// gnostic parser just doesn't support more
// than one Schema here
v.Schema = &Schema{}
c.Fuzz(&v.Schema)
},
func(v *SchemaOrBool, c fuzz.Continue) {
*v = SchemaOrBool{}
if c.RandBool() {
v.Allows = c.RandBool()
} else {
v.Schema = &Schema{}
v.Allows = true
c.Fuzz(&v.Schema)
}
},
func(v map[string]Response, c fuzz.Continue) {
n := 0
c.Fuzz(&n)
if n == 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
return
}
// Response definitions are not allowed to
// be refs
for i := 0; i < c.Intn(5)+1; i++ {
resp := &Response{}
c.Fuzz(resp)
resp.Ref = Ref{}
resp.Description = c.RandString() + "x"
// Response refs are not vendor extensible by gnostic
resp.VendorExtensible.Extensions = nil
v[c.RandString()+"x"] = *resp
}
},
func(v *Header, c fuzz.Continue) {
if v != nil {
c.FuzzNoCustom(v)
// descendant Items of Header may not be refs
cur := v.Items
for cur != nil {
cur.Ref = Ref{}
cur = cur.Items
}
}
},
func(v *Ref, c fuzz.Continue) {
*v = Ref{}
v.Ref, _ = jsonreference.New("http://asd.com/" + c.RandString())
},
func(v *Response, c fuzz.Continue) {
*v = Response{}
if c.RandBool() {
v.Ref = Ref{}
v.Ref.Ref, _ = jsonreference.New("http://asd.com/" + c.RandString())
} else {
c.Fuzz(&v.VendorExtensible)
c.Fuzz(&v.Schema)
c.Fuzz(&v.ResponseProps)
v.Headers = nil
v.Ref = Ref{}
n := 0
c.Fuzz(&n)
if n != 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
num := c.Intn(4)
for i := 0; i < num; i++ {
if v.Headers == nil {
v.Headers = make(map[string]Header)
}
hdr := Header{}
c.Fuzz(&hdr)
if hdr.Type == "" {
// hit maxDepth, just abort trying to make haders
v.Headers = nil
break
}
v.Headers[c.RandString()+"x"] = hdr
}
} else {
v.Headers = nil
}
}
v.Description = c.RandString() + "x"
// Gnostic parses empty as nil, so to keep avoid putting empty
if len(v.Headers) == 0 {
v.Headers = nil
}
},
func(v **Info, c fuzz.Continue) {
// Info is never nil
*v = &Info{}
c.FuzzNoCustom(*v)
(*v).Title = c.RandString() + "x"
},
func(v *Extensions, c fuzz.Continue) {
// gnostic parser only picks up x- vendor extensions
numChildren := c.Intn(5)
for i := 0; i < numChildren; i++ {
if *v == nil {
*v = Extensions{}
}
(*v)["x-"+c.RandString()] = c.RandString()
}
},
func(v *Swagger, c fuzz.Continue) {
c.FuzzNoCustom(v)
if v.Paths == nil {
// Force paths non-nil since it does not have omitempty in json tag.
// This means a perfect roundtrip (via json) is impossible,
// since we can't tell the difference between empty/unspecified paths
v.Paths = &Paths{}
c.Fuzz(v.Paths)
}
v.Swagger = "2.0"
// Gnostic support serializing ID at all
// unavoidable data loss
v.ID = ""
v.Schemes = nil
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "http")
}
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "https")
}
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "ws")
}
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "wss")
}
// Gnostic unconditionally makes security values non-null
// So do not fuzz null values into the array.
for i, val := range v.Security {
if val == nil {
v.Security[i] = make(map[string][]string)
}
for k, v := range val {
if v == nil {
val[k] = make([]string, 0)
}
}
}
},
func(v *SecurityScheme, c fuzz.Continue) {
v.Description = c.RandString() + "x"
c.Fuzz(&v.VendorExtensible)
switch c.Intn(3) {
case 0:
v.Type = "basic"
case 1:
v.Type = "apiKey"
switch c.Intn(2) {
case 0:
v.In = "header"
case 1:
v.In = "query"
default:
panic("unreachable")
}
v.Name = "x" + c.RandString()
case 2:
v.Type = "oauth2"
switch c.Intn(4) {
case 0:
v.Flow = "accessCode"
v.TokenURL = "https://" + c.RandString()
v.AuthorizationURL = "https://" + c.RandString()
case 1:
v.Flow = "application"
v.TokenURL = "https://" + c.RandString()
case 2:
v.Flow = "implicit"
v.AuthorizationURL = "https://" + c.RandString()
case 3:
v.Flow = "password"
v.TokenURL = "https://" + c.RandString()
default:
panic("unreachable")
}
c.Fuzz(&v.Scopes)
default:
panic("unreachable")
}
},
func(v *interface{}, c fuzz.Continue) {
*v = c.RandString() + "x"
},
func(v *string, c fuzz.Continue) {
*v = c.RandString() + "x"
},
func(v *ExternalDocumentation, c fuzz.Continue) {
v.Description = c.RandString() + "x"
v.URL = c.RandString() + "x"
},
func(v *SimpleSchema, c fuzz.Continue) {
c.FuzzNoCustom(v)
switch c.Intn(5) {
case 0:
v.Type = "string"
case 1:
v.Type = "number"
case 2:
v.Type = "boolean"
case 3:
v.Type = "integer"
case 4:
v.Type = "array"
default:
panic("unreachable")
}
switch c.Intn(5) {
case 0:
v.CollectionFormat = "csv"
case 1:
v.CollectionFormat = "ssv"
case 2:
v.CollectionFormat = "tsv"
case 3:
v.CollectionFormat = "pipes"
case 4:
v.CollectionFormat = ""
default:
panic("unreachable")
}
// None of the types which include SimpleSchema in our definitions
// actually support "example" in the official spec
v.Example = nil
// unsupported by openapi
v.Nullable = false
},
func(v *int64, c fuzz.Continue) {
c.Fuzz(v)
// Gnostic does not differentiate between 0 and non-specified
// so avoid using 0 for fuzzer
if *v == 0 {
*v = 1
}
},
func(v *float64, c fuzz.Continue) {
c.Fuzz(v)
// Gnostic does not differentiate between 0 and non-specified
// so avoid using 0 for fuzzer
if *v == 0.0 {
*v = 1.0
}
},
func(v *Parameter, c fuzz.Continue) {
if v == nil {
return
}
c.Fuzz(&v.VendorExtensible)
if c.RandBool() {
// body param
v.Description = c.RandString() + "x"
v.Name = c.RandString() + "x"
v.In = "body"
c.Fuzz(&v.Description)
c.Fuzz(&v.Required)
v.Schema = &Schema{}
c.Fuzz(&v.Schema)
} else {
c.Fuzz(&v.SimpleSchema)
c.Fuzz(&v.CommonValidations)
v.AllowEmptyValue = false
v.Description = c.RandString() + "x"
v.Name = c.RandString() + "x"
switch c.Intn(4) {
case 0:
// Header param
v.In = "header"
case 1:
// Form data param
v.In = "formData"
v.AllowEmptyValue = c.RandBool()
case 2:
// Query param
v.In = "query"
v.AllowEmptyValue = c.RandBool()
case 3:
// Path param
v.In = "path"
v.Required = true
default:
panic("unreachable")
}
// descendant Items of Parameter may not be refs
cur := v.Items
for cur != nil {
cur.Ref = Ref{}
cur = cur.Items
}
}
},
func(v *Schema, c fuzz.Continue) {
if c.RandBool() {
// file schema
c.Fuzz(&v.Default)
c.Fuzz(&v.Description)
c.Fuzz(&v.Example)
c.Fuzz(&v.ExternalDocs)
c.Fuzz(&v.Format)
c.Fuzz(&v.ReadOnly)
c.Fuzz(&v.Required)
c.Fuzz(&v.Title)
v.Type = StringOrArray{"file"}
} else {
// normal schema
c.Fuzz(&v.SchemaProps)
c.Fuzz(&v.SwaggerSchemaProps)
c.Fuzz(&v.VendorExtensible)
// c.Fuzz(&v.ExtraProps)
// ExtraProps will not roundtrip - gnostic throws out
// unrecognized keys
}
// Not supported by official openapi v2 spec
// and stripped by k8s apiserver
v.ID = ""
v.AnyOf = nil
v.OneOf = nil
v.Not = nil
v.Nullable = false
v.AdditionalItems = nil
v.Schema = ""
v.PatternProperties = nil
v.Definitions = nil
v.Dependencies = nil
},
}
var SwaggerDiffOptions = []cmp.Option{
// cmp.Diff panics on Ref since jsonreference.Ref uses unexported fields
cmp.Comparer(func(a Ref, b Ref) bool {
return a.String() == b.String()
}),
}

View File

@ -219,8 +219,8 @@ func (k *Ref) FromGnostic(g string) error {
// Caveats:
//
// - gnostic v2 documents treats zero as unspecified for numerical fields of
//CommonValidations fields such as Maximum, Minimum, MaximumItems, etc.
//There will always be data loss if one of the values of these fields is set to zero.
// CommonValidations fields such as Maximum, Minimum, MaximumItems, etc.
// There will always be data loss if one of the values of these fields is set to zero.
//
// Returns:
//
@ -1263,6 +1263,8 @@ func (k *Schema) FromGnostic(g *openapi_v2.Schema) (ok bool, err error) {
k.AdditionalProperties.Allows = g.AdditionalProperties.GetBoolean()
} else {
k.AdditionalProperties.Schema = &Schema{}
k.AdditionalProperties.Allows = true
if nok, err := k.AdditionalProperties.Schema.FromGnostic(g.AdditionalProperties.GetSchema()); err != nil {
return false, err
} else if !nok {

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
const (
@ -62,6 +64,10 @@ func (h Header) MarshalJSON() ([]byte, error) {
// UnmarshalJSON unmarshals this header from JSON
func (h *Header) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, h)
}
if err := json.Unmarshal(data, &h.CommonValidations); err != nil {
return err
}
@ -73,3 +79,27 @@ func (h *Header) UnmarshalJSON(data []byte) error {
}
return json.Unmarshal(data, &h.HeaderProps)
}
func (h *Header) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
CommonValidations
SimpleSchema
Extensions
HeaderProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
h.CommonValidations = x.CommonValidations
h.SimpleSchema = x.SimpleSchema
h.Extensions = x.Extensions
h.HeaderProps = x.HeaderProps
h.Extensions.sanitize()
if len(h.Extensions) == 0 {
h.Extensions = nil
}
return nil
}

View File

@ -19,6 +19,8 @@ import (
"strings"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Extensions vendor specific extensions
@ -87,6 +89,31 @@ func (e Extensions) GetObject(key string, out interface{}) error {
return nil
}
func (e Extensions) sanitize() {
for k := range e {
if !isExtensionKey(k) {
delete(e, k)
}
}
}
func (e Extensions) sanitizeWithExtra() (extra map[string]any) {
for k, v := range e {
if !isExtensionKey(k) {
if extra == nil {
extra = make(map[string]any)
}
extra[k] = v
delete(e, k)
}
}
return extra
}
func isExtensionKey(k string) bool {
return len(k) > 1 && (k[0] == 'x' || k[0] == 'X') && k[1] == '-'
}
// VendorExtensible composition block.
type VendorExtensible struct {
Extensions Extensions
@ -167,8 +194,29 @@ func (i Info) MarshalJSON() ([]byte, error) {
// UnmarshalJSON marshal this from JSON
func (i *Info) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, i)
}
if err := json.Unmarshal(data, &i.InfoProps); err != nil {
return err
}
return json.Unmarshal(data, &i.VendorExtensible)
}
func (i *Info) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
InfoProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
i.VendorExtensible.Extensions = x.Extensions
i.InfoProps = x.InfoProps
return nil
}

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
const (
@ -64,6 +66,10 @@ type Items struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (i *Items) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, i)
}
var validations CommonValidations
if err := json.Unmarshal(data, &validations); err != nil {
return err
@ -87,6 +93,28 @@ func (i *Items) UnmarshalJSON(data []byte) error {
return nil
}
func (i *Items) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
CommonValidations
SimpleSchema
Extensions
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
if err := i.Refable.Ref.fromMap(x.Extensions); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
i.CommonValidations = x.CommonValidations
i.SimpleSchema = x.SimpleSchema
i.VendorExtensible.Extensions = x.Extensions
return nil
}
// MarshalJSON converts this items object to JSON
func (i Items) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(i.CommonValidations)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// OperationProps describes an operation
@ -75,12 +77,34 @@ type Operation struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (o *Operation) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, o)
}
if err := json.Unmarshal(data, &o.OperationProps); err != nil {
return err
}
return json.Unmarshal(data, &o.VendorExtensible)
}
func (o *Operation) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
type OperationPropsNoMethods OperationProps // strip MarshalJSON method
var x struct {
Extensions
OperationPropsNoMethods
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
o.VendorExtensible.Extensions = x.Extensions
o.OperationProps = OperationProps(x.OperationPropsNoMethods)
return nil
}
// MarshalJSON converts this items object to JSON
func (o Operation) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(o.OperationProps)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// ParamProps describes the specific attributes of an operation parameter
@ -38,26 +40,31 @@ type ParamProps struct {
//
// There are five possible parameter types.
// * Path - Used together with [Path Templating](#pathTemplating), where the parameter value is actually part
// of the operation's URL. This does not include the host or base path of the API. For example, in `/items/{itemId}`,
// the path parameter is `itemId`.
//
// of the operation's URL. This does not include the host or base path of the API. For example, in `/items/{itemId}`,
// the path parameter is `itemId`.
//
// * Query - Parameters that are appended to the URL. For example, in `/items?id=###`, the query parameter is `id`.
// * Header - Custom headers that are expected as part of the request.
// * Body - The payload that's appended to the HTTP request. Since there can only be one payload, there can only be
// _one_ body parameter. The name of the body parameter has no effect on the parameter itself and is used for
// documentation purposes only. Since Form parameters are also in the payload, body and form parameters cannot exist
// together for the same operation.
//
// _one_ body parameter. The name of the body parameter has no effect on the parameter itself and is used for
// documentation purposes only. Since Form parameters are also in the payload, body and form parameters cannot exist
// together for the same operation.
//
// * Form - Used to describe the payload of an HTTP request when either `application/x-www-form-urlencoded` or
// `multipart/form-data` are used as the content type of the request (in Swagger's definition,
// the [`consumes`](#operationConsumes) property of an operation). This is the only parameter type that can be used
// to send files, thus supporting the `file` type. Since form parameters are sent in the payload, they cannot be
// declared together with a body parameter for the same operation. Form parameters have a different format based on
// the content-type used (for further details, consult http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4).
// * `application/x-www-form-urlencoded` - Similar to the format of Query parameters but as a payload.
// For example, `foo=1&bar=swagger` - both `foo` and `bar` are form parameters. This is normally used for simple
// parameters that are being transferred.
// * `multipart/form-data` - each parameter takes a section in the payload with an internal header.
// For example, for the header `Content-Disposition: form-data; name="submit-name"` the name of the parameter is
// `submit-name`. This type of form parameters is more commonly used for file transfers.
//
// `multipart/form-data` are used as the content type of the request (in Swagger's definition,
// the [`consumes`](#operationConsumes) property of an operation). This is the only parameter type that can be used
// to send files, thus supporting the `file` type. Since form parameters are sent in the payload, they cannot be
// declared together with a body parameter for the same operation. Form parameters have a different format based on
// the content-type used (for further details, consult http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4).
// * `application/x-www-form-urlencoded` - Similar to the format of Query parameters but as a payload.
// For example, `foo=1&bar=swagger` - both `foo` and `bar` are form parameters. This is normally used for simple
// parameters that are being transferred.
// * `multipart/form-data` - each parameter takes a section in the payload with an internal header.
// For example, for the header `Content-Disposition: form-data; name="submit-name"` the name of the parameter is
// `submit-name`. This type of form parameters is more commonly used for file transfers.
//
// For more information: http://goo.gl/8us55a#parameterObject
type Parameter struct {
@ -70,6 +77,10 @@ type Parameter struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (p *Parameter) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, p)
}
if err := json.Unmarshal(data, &p.CommonValidations); err != nil {
return err
}
@ -85,6 +96,30 @@ func (p *Parameter) UnmarshalJSON(data []byte) error {
return json.Unmarshal(data, &p.ParamProps)
}
func (p *Parameter) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
CommonValidations
SimpleSchema
Extensions
ParamProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
if err := p.Refable.Ref.fromMap(x.Extensions); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
p.CommonValidations = x.CommonValidations
p.SimpleSchema = x.SimpleSchema
p.VendorExtensible.Extensions = x.Extensions
p.ParamProps = x.ParamProps
return nil
}
// MarshalJSON converts this items object to JSON
func (p Parameter) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(p.CommonValidations)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// PathItemProps the path item specific properties
@ -46,6 +48,10 @@ type PathItem struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (p *PathItem) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, p)
}
if err := json.Unmarshal(data, &p.Refable); err != nil {
return err
}
@ -55,6 +61,31 @@ func (p *PathItem) UnmarshalJSON(data []byte) error {
return json.Unmarshal(data, &p.PathItemProps)
}
func (p *PathItem) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
PathItemProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
p.Extensions = x.Extensions
p.PathItemProps = x.PathItemProps
if err := p.Refable.Ref.fromMap(p.Extensions); err != nil {
return err
}
p.Extensions.sanitize()
if len(p.Extensions) == 0 {
p.Extensions = nil
}
return nil
}
// MarshalJSON converts this items object to JSON
func (p PathItem) MarshalJSON() ([]byte, error) {
b3, err := json.Marshal(p.Refable)

View File

@ -16,9 +16,12 @@ package spec
import (
"encoding/json"
"fmt"
"strings"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Paths holds the relative paths to the individual endpoints.
@ -34,6 +37,10 @@ type Paths struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (p *Paths) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, p)
}
var res map[string]json.RawMessage
if err := json.Unmarshal(data, &res); err != nil {
return err
@ -63,6 +70,58 @@ func (p *Paths) UnmarshalJSON(data []byte) error {
return nil
}
func (p *Paths) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
tok, err := dec.ReadToken()
if err != nil {
return err
}
var ext any
var pi PathItem
switch k := tok.Kind(); k {
case 'n':
return nil // noop
case '{':
for {
tok, err := dec.ReadToken()
if err != nil {
return err
}
if tok.Kind() == '}' {
return nil
}
switch k := tok.String(); {
case isExtensionKey(k):
ext = nil
if err := opts.UnmarshalNext(dec, &ext); err != nil {
return err
}
if p.Extensions == nil {
p.Extensions = make(map[string]any)
}
p.Extensions[k] = ext
case len(k) > 0 && k[0] == '/':
pi = PathItem{}
if err := opts.UnmarshalNext(dec, &pi); err != nil {
return err
}
if p.Paths == nil {
p.Paths = make(map[string]PathItem)
}
p.Paths[k] = pi
default:
_, err := dec.ReadValue() // skip value
return err
}
}
default:
return fmt.Errorf("unknown JSON kind: %v", k)
}
}
// MarshalJSON converts this items object to JSON
func (p Paths) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(p.VendorExtensible)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// ResponseProps properties specific to a response
@ -39,13 +41,46 @@ type Response struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (r *Response) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, r)
}
if err := json.Unmarshal(data, &r.ResponseProps); err != nil {
return err
}
if err := json.Unmarshal(data, &r.Refable); err != nil {
return err
}
return json.Unmarshal(data, &r.VendorExtensible)
if err := json.Unmarshal(data, &r.VendorExtensible); err != nil {
return err
}
return nil
}
func (r *Response) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
ResponseProps
Extensions
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
r.Extensions = x.Extensions
r.ResponseProps = x.ResponseProps
if err := r.Refable.Ref.fromMap(r.Extensions); err != nil {
return err
}
r.Extensions.sanitize()
if len(r.Extensions) == 0 {
r.Extensions = nil
}
return nil
}
// MarshalJSON converts this items object to JSON

View File

@ -16,10 +16,13 @@ package spec
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Responses is a container for the expected responses of an operation.
@ -42,6 +45,10 @@ type Responses struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (r *Responses) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, r)
}
if err := json.Unmarshal(data, &r.ResponsesProps); err != nil {
return err
}
@ -90,21 +97,90 @@ func (r ResponsesProps) MarshalJSON() ([]byte, error) {
// UnmarshalJSON unmarshals responses from JSON
func (r *ResponsesProps) UnmarshalJSON(data []byte) error {
var res map[string]Response
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, r)
}
var res map[string]json.RawMessage
if err := json.Unmarshal(data, &res); err != nil {
return nil
return err
}
if v, ok := res["default"]; ok {
r.Default = &v
value := Response{}
if err := json.Unmarshal(v, &value); err != nil {
return err
}
r.Default = &value
delete(res, "default")
}
for k, v := range res {
// Take all integral keys
if nk, err := strconv.Atoi(k); err == nil {
if r.StatusCodeResponses == nil {
r.StatusCodeResponses = map[int]Response{}
}
r.StatusCodeResponses[nk] = v
value := Response{}
if err := json.Unmarshal(v, &value); err != nil {
return err
}
r.StatusCodeResponses[nk] = value
}
}
return nil
}
func (r *Responses) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) (err error) {
tok, err := dec.ReadToken()
if err != nil {
return err
}
var ext any
var resp Response
switch k := tok.Kind(); k {
case 'n':
return nil // noop
case '{':
for {
tok, err := dec.ReadToken()
if err != nil {
return err
}
if tok.Kind() == '}' {
return nil
}
switch k := tok.String(); {
case isExtensionKey(k):
ext = nil
if err := opts.UnmarshalNext(dec, &ext); err != nil {
return err
}
if r.Extensions == nil {
r.Extensions = make(map[string]any)
}
r.Extensions[k] = ext
case k == "default":
resp = Response{}
if err := opts.UnmarshalNext(dec, &resp); err != nil {
return err
}
respCopy := resp
r.ResponsesProps.Default = &respCopy
default:
if nk, err := strconv.Atoi(k); err == nil {
resp = Response{}
if err := opts.UnmarshalNext(dec, &resp); err != nil {
return err
}
if r.StatusCodeResponses == nil {
r.StatusCodeResponses = map[int]Response{}
}
r.StatusCodeResponses[nk] = resp
}
}
}
default:
return fmt.Errorf("unknown JSON kind: %v", k)
}
}

View File

@ -21,6 +21,8 @@ import (
"strings"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// BooleanProperty creates a boolean property
@ -465,6 +467,10 @@ func (s Schema) MarshalJSON() ([]byte, error) {
// UnmarshalJSON marshal this from JSON
func (s *Schema) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
props := struct {
SchemaProps
SwaggerSchemaProps
@ -511,3 +517,38 @@ func (s *Schema) UnmarshalJSON(data []byte) error {
return nil
}
func (s *Schema) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
SchemaProps
SwaggerSchemaProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
if err := x.Ref.fromMap(x.Extensions); err != nil {
return err
}
if err := x.Schema.fromMap(x.Extensions); err != nil {
return err
}
delete(x.Extensions, "$ref")
delete(x.Extensions, "$schema")
for _, pn := range swag.DefaultJSONNameProvider.GetJSONNames(s) {
delete(x.Extensions, pn)
}
if len(x.Extensions) == 0 {
x.Extensions = nil
}
s.ExtraProps = x.Extensions.sanitizeWithExtra()
s.VendorExtensible.Extensions = x.Extensions
s.SchemaProps = x.SchemaProps
s.SwaggerSchemaProps = x.SwaggerSchemaProps
return nil
}

View File

@ -18,6 +18,7 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// SecuritySchemeProps describes a swagger security scheme in the securityDefinitions section
@ -62,3 +63,20 @@ func (s *SecurityScheme) UnmarshalJSON(data []byte) error {
}
return json.Unmarshal(data, &s.VendorExtensible)
}
func (s *SecurityScheme) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
SecuritySchemeProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
s.VendorExtensible.Extensions = x.Extensions
s.SecuritySchemeProps = x.SecuritySchemeProps
return nil
}

View File

@ -19,6 +19,8 @@ import (
"fmt"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Swagger this is the root document object for the API specification.
@ -46,6 +48,10 @@ func (s Swagger) MarshalJSON() ([]byte, error) {
// UnmarshalJSON unmarshals a swagger spec from json
func (s *Swagger) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var sw Swagger
if err := json.Unmarshal(data, &sw.SwaggerProps); err != nil {
return err
@ -57,6 +63,30 @@ func (s *Swagger) UnmarshalJSON(data []byte) error {
return nil
}
func (s *Swagger) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
// Note: If you're willing to make breaking changes, it is possible to
// optimize this and other usages of this pattern:
// https://github.com/kubernetes/kube-openapi/pull/319#discussion_r983165948
var x struct {
Extensions
SwaggerProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
s.Extensions = x.Extensions
s.SwaggerProps = x.SwaggerProps
s.Extensions.sanitize()
if len(s.Extensions) == 0 {
s.Extensions = nil
}
return nil
}
// SwaggerProps captures the top-level properties of an Api specification
//
// NOTE: validation rules
@ -108,6 +138,10 @@ func (s SchemaOrBool) MarshalJSON() ([]byte, error) {
// UnmarshalJSON converts this bool or schema object from a JSON structure
func (s *SchemaOrBool) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var nw SchemaOrBool
if len(data) >= 4 {
if data[0] == '{' {
@ -123,6 +157,26 @@ func (s *SchemaOrBool) UnmarshalJSON(data []byte) error {
return nil
}
func (s *SchemaOrBool) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch k := dec.PeekKind(); k {
case '{':
err := opts.UnmarshalNext(dec, &s.Schema)
if err != nil {
return err
}
s.Allows = true
return nil
case 't', 'f':
err := opts.UnmarshalNext(dec, &s.Allows)
if err != nil {
return err
}
return nil
default:
return fmt.Errorf("expected object or bool, not '%v'", k.String())
}
}
// SchemaOrStringArray represents a schema or a string array
type SchemaOrStringArray struct {
Schema *Schema
@ -142,6 +196,10 @@ func (s SchemaOrStringArray) MarshalJSON() ([]byte, error) {
// UnmarshalJSON converts this schema object or array from a JSON structure
func (s *SchemaOrStringArray) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var first byte
if len(data) > 1 {
first = data[0]
@ -163,6 +221,18 @@ func (s *SchemaOrStringArray) UnmarshalJSON(data []byte) error {
return nil
}
func (s *SchemaOrStringArray) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch dec.PeekKind() {
case '{':
return opts.UnmarshalNext(dec, &s.Schema)
case '[':
return opts.UnmarshalNext(dec, &s.Property)
default:
_, err := dec.ReadValue()
return err
}
}
// Definitions contains the models explicitly defined in this spec
// An object to hold data types that can be consumed and produced by operations.
// These data types can be primitives, arrays or models.
@ -193,6 +263,10 @@ func (s StringOrArray) Contains(value string) bool {
// UnmarshalJSON unmarshals this string or array object from a JSON array or JSON string
func (s *StringOrArray) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var first byte
if len(data) > 1 {
first = data[0]
@ -223,6 +297,23 @@ func (s *StringOrArray) UnmarshalJSON(data []byte) error {
}
}
func (s *StringOrArray) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch k := dec.PeekKind(); k {
case '[':
*s = StringOrArray{}
return opts.UnmarshalNext(dec, (*[]string)(s))
case '"':
*s = StringOrArray{""}
return opts.UnmarshalNext(dec, &(*s)[0])
case 'n':
// Throw out null token
_, _ = dec.ReadToken()
return nil
default:
return fmt.Errorf("expected string or array, not '%v'", k.String())
}
}
// MarshalJSON converts this string or array to a JSON array or JSON string
func (s StringOrArray) MarshalJSON() ([]byte, error) {
if len(s) == 1 {
@ -264,6 +355,10 @@ func (s SchemaOrArray) MarshalJSON() ([]byte, error) {
// UnmarshalJSON converts this schema object or array from a JSON structure
func (s *SchemaOrArray) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var nw SchemaOrArray
var first byte
if len(data) > 1 {
@ -284,3 +379,15 @@ func (s *SchemaOrArray) UnmarshalJSON(data []byte) error {
*s = nw
return nil
}
func (s *SchemaOrArray) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch dec.PeekKind() {
case '{':
return opts.UnmarshalNext(dec, &s.Schema)
case '[':
return opts.UnmarshalNext(dec, &s.Schemas)
default:
_, err := dec.ReadValue()
return err
}
}

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// TagProps describe a tag entry in the top level tags section of a swagger spec
@ -52,8 +54,29 @@ func (t Tag) MarshalJSON() ([]byte, error) {
// UnmarshalJSON marshal this from JSON
func (t *Tag) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, t)
}
if err := json.Unmarshal(data, &t.TagProps); err != nil {
return err
}
return json.Unmarshal(data, &t.VendorExtensible)
}
func (t *Tag) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
TagProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
t.VendorExtensible.Extensions = x.Extensions
t.TagProps = x.TagProps
return nil
}

View File

@ -67,11 +67,6 @@ type ChannelableStatus struct {
// resolved delivery options.
// +optional
DeliveryStatus `json:",inline"`
// DeadLetterChannel is a KReference and is set by the channel when it supports native error handling via a channel
// Failed messages are delivered here.
// Deprecated in favor of DeliveryStatus, to be removed September 2022.
// +optional
DeadLetterChannel *duckv1.KReference `json:"deadLetterChannel,omitempty"`
}
var (

View File

@ -117,11 +117,6 @@ func (in *ChannelableStatus) DeepCopyInto(out *ChannelableStatus) {
in.AddressStatus.DeepCopyInto(&out.AddressStatus)
in.SubscribableStatus.DeepCopyInto(&out.SubscribableStatus)
in.DeliveryStatus.DeepCopyInto(&out.DeliveryStatus)
if in.DeadLetterChannel != nil {
in, out := &in.DeadLetterChannel, &out.DeadLetterChannel
*out = new(duckv1.KReference)
**out = **in
}
return
}

View File

@ -34,5 +34,8 @@ func (ss *SubscriptionSpec) SetDefaults(ctx context.Context) {
if ss == nil {
return
}
ss.Subscriber.SetDefaults(ctx)
ss.Reply.SetDefaults(ctx)
ss.Delivery.SetDefaults(ctx)
}

View File

@ -111,6 +111,7 @@ export KO_DOCKER_REPO="gcr.io/knative-nightly"
# Build stripped binary to reduce size
export GOFLAGS="-ldflags=-s -ldflags=-w"
export GITHUB_TOKEN=""
readonly IMAGES_REFS_FILE="${IMAGES_REFS_FILE:-$(mktemp -d)/images_refs.txt}"
# Convenience function to run the hub tool.
# Parameters: $1..$n - arguments to hub.
@ -313,40 +314,89 @@ function build_from_source() {
}
function get_images_in_yamls() {
rm -rf imagerefs.txt
rm -rf "$IMAGES_REFS_FILE"
echo "Assembling a list of image refences to sign"
for file in $@; do
for file in "$@"; do
[[ "${file##*.}" != "yaml" ]] && continue
echo "Inspecting ${file}"
for image in $(grep -oh "\S*${KO_DOCKER_REPO}\S*" "${file}"); do
echo $image >> imagerefs.txt
done
while read -r image; do
echo "$image" >> "$IMAGES_REFS_FILE"
done < <(grep -oh "\S*${KO_DOCKER_REPO}\S*" "${file}")
done
sort -uo imagerefs.txt imagerefs.txt # Remove duplicate entries
if [[ -f "$IMAGES_REFS_FILE" ]]; then
sort -uo "$IMAGES_REFS_FILE" "$IMAGES_REFS_FILE" # Remove duplicate entries
fi
}
# Finds a checksums file within the given list of artifacts (space delimited)
# Parameters: $n - artifact files
function find_checksums_file() {
for arg in "$@"; do
# kinda dirty hack needed as we pass $ARTIFACTS_TO_PUBLISH in space
# delimiter variable, which is vulnerable to all sorts of argument quoting
while read -r file; do
if [[ "${file}" == *"checksums.txt" ]]; then
echo "${file}"
return 0
fi
done < <(echo "$arg" | tr ' ' '\n')
done
warning "cannot find checksums file"
}
# Build a release from source.
function sign_release() {
get_images_in_yamls "${ARTIFACTS_TO_PUBLISH}"
if (( ! IS_PROW )); then # This function can't be run by devs on their laptops
return 0
fi
get_images_in_yamls "${ARTIFACTS_TO_PUBLISH}"
local checksums_file
checksums_file="$(find_checksums_file "${ARTIFACTS_TO_PUBLISH}")"
if ! [[ -f "${checksums_file}" ]]; then
echo '>> No checksums file found, generating one'
checksums_file="$(mktemp -d)/checksums.txt"
for file in ${ARTIFACTS_TO_PUBLISH}; do
pushd "$(dirname "$file")" >/dev/null
sha256sum "$(basename "$file")" >> "${checksums_file}"
popd >/dev/null
done
ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} ${checksums_file}"
fi
# Notarizing mac binaries needs to be done before cosign as it changes the checksum values
# of the darwin binaries
if [ -n "${APPLE_CODESIGN_KEY}" ] && [ -n "${APPLE_CODESIGN_PASSWORD_FILE}" ] && [ -n "${APPLE_NOTARY_API_KEY}" ]; then
banner "Notarizing macOS Binaries for the release"
FILES=$(find -- * -type f -name "*darwin*")
for file in $FILES; do
rcodesign sign "${file}" --p12-file="${APPLE_CODESIGN_KEY}" \
--code-signature-flags=runtime \
--p12-password-file="${APPLE_CODESIGN_PASSWORD_FILE}"
done
zip files.zip ${FILES}
rcodesign notary-submit files.zip --api-key-path="${APPLE_NOTARY_API_KEY}" --wait
sha256sum ${ARTIFACTS_TO_PUBLISH//checksums.txt/} > checksums.txt
echo "🧮 Post Notarization Checksum:"
cat checksums.txt
local macos_artifacts
declare -a macos_artifacts=()
while read -r file; do
if echo "$file" | grep -q "darwin"; then
macos_artifacts+=("${file}")
rcodesign sign "${file}" --p12-file="${APPLE_CODESIGN_KEY}" \
--code-signature-flags=runtime \
--p12-password-file="${APPLE_CODESIGN_PASSWORD_FILE}"
fi
done < <(echo "${ARTIFACTS_TO_PUBLISH}" | tr ' ' '\n')
if [[ -z "${macos_artifacts[*]}" ]]; then
warning "No macOS binaries found, skipping notarization"
else
local zip_file
zip_file="$(mktemp -d)/files.zip"
zip "$zip_file" -@ < <(printf "%s\n" "${macos_artifacts[@]}")
rcodesign notary-submit "$zip_file" --api-key-path="${APPLE_NOTARY_API_KEY}" --wait
true > "${checksums_file}" # Clear the checksums file
for file in ${ARTIFACTS_TO_PUBLISH}; do
if echo "$file" | grep -q "checksums.txt"; then
continue # Don't checksum the checksums file
fi
pushd "$(dirname "$file")" >/dev/null
sha256sum "$(basename "$file")" >> "${checksums_file}"
popd >/dev/null
done
echo "🧮 Post Notarization Checksum:"
cat "$checksums_file"
fi
fi
ID_TOKEN=$(gcloud auth print-identity-token --audiences=sigstore \
@ -354,23 +404,25 @@ function sign_release() {
--impersonate-service-account="${SIGNING_IDENTITY}")
echo "Signing Images with the identity ${SIGNING_IDENTITY}"
## Sign the images with cosign
if [[ -f "imagerefs.txt" ]]; then
COSIGN_EXPERIMENTAL=1 cosign sign $(cat imagerefs.txt) --recursive --identity-token="${ID_TOKEN}"
if [ -n "${ATTEST_IMAGES:-}" ]; then # Temporary Feature Gate
provenance-generator --clone-log=/logs/clone.json \
--image-refs=imagerefs.txt --output=attestation.json
mkdir -p "${ARTIFACTS}"/attestation && cp attestation.json "${ARTIFACTS}"/attestation
COSIGN_EXPERIMENTAL=1 cosign attest $(cat imagerefs.txt) --recursive --identity-token="${ID_TOKEN}" \
--predicate=attestation.json --type=slsaprovenance
fi
if [[ -f "$IMAGES_REFS_FILE" ]]; then
COSIGN_EXPERIMENTAL=1 cosign sign $(cat "$IMAGES_REFS_FILE") \
--recursive --identity-token="${ID_TOKEN}"
if [ -n "${ATTEST_IMAGES:-}" ]; then # Temporary Feature Gate
provenance-generator --clone-log=/logs/clone.json \
--image-refs="$IMAGES_REFS_FILE" --output=attestation.json
mkdir -p "${ARTIFACTS}"/attestation && cp attestation.json "${ARTIFACTS}"/attestation
COSIGN_EXPERIMENTAL=1 cosign attest $(cat "$IMAGES_REFS_FILE") \
--recursive --identity-token="${ID_TOKEN}" \
--predicate=attestation.json --type=slsaprovenance
fi
fi
## Check if there is checksums.txt file. If so, sign the checksum file
if [[ -f "checksums.txt" ]]; then
echo "Signing Images with the identity ${SIGNING_IDENTITY}"
COSIGN_EXPERIMENTAL=1 cosign sign-blob checksums.txt --output-signature=checksums.txt.sig --output-certificate=checksums.txt.pem --identity-token="${ID_TOKEN}"
ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} checksums.txt.sig checksums.txt.pem"
fi
echo "Signing checksums with the identity ${SIGNING_IDENTITY}"
COSIGN_EXPERIMENTAL=1 cosign sign-blob "$checksums_file" \
--output-signature="${checksums_file}.sig" \
--output-certificate="${checksums_file}.pem" \
--identity-token="${ID_TOKEN}"
ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} ${checksums_file}.sig ${checksums_file}.pem"
}
# Copy tagged images from the nightly GCR to the release GCR, tagging them 'latest'.
@ -678,7 +730,7 @@ function main() {
# Parameters: $1..$n - files to add to the release.
function publish_to_github() {
(( PUBLISH_TO_GITHUB )) || return 0
local title="${REPO_NAME_FORMATTED} release ${TAG}"
local title="${TAG}"
local attachments=()
local description="$(mktemp)"
local attachments_dir="$(mktemp -d)"

View File

@ -73,6 +73,10 @@ func (d *Destination) GetRef() *KReference {
}
func (d *Destination) SetDefaults(ctx context.Context) {
if d == nil {
return
}
if d.Ref != nil && d.Ref.Namespace == "" {
d.Ref.Namespace = apis.ParentMeta(ctx).Namespace
}

View File

@ -230,10 +230,15 @@ func NewImpl(ctx {{.contextContext|raw}}, r Interface{{if .hasClass}}, classValu
lister := {{.type|lowercaseSingular}}Informer.Lister()
var promoteFilterFunc func(obj interface{}) bool
var promoteFunc = func(bkt {{.reconcilerBucket|raw}}) {}
rec := &reconcilerImpl{
LeaderAwareFuncs: {{.reconcilerLeaderAwareFuncs|raw}}{
PromoteFunc: func(bkt {{.reconcilerBucket|raw}}, enq func({{.reconcilerBucket|raw}}, {{.typesNamespacedName|raw}})) error {
// Signal promotion event
promoteFunc(bkt)
all, err := lister.List({{.labelsEverything|raw}}())
if err != nil {
return err
@ -295,6 +300,9 @@ func NewImpl(ctx {{.contextContext|raw}}, r Interface{{if .hasClass}}, classValu
if opts.PromoteFilterFunc != nil {
promoteFilterFunc = opts.PromoteFilterFunc
}
if opts.PromoteFunc != nil {
promoteFunc = opts.PromoteFunc
}
}
rec.Recorder = createRecorder(ctx, agentName)

View File

@ -150,6 +150,10 @@ func (g *reconcilerReconcilerGenerator) GenerateType(c *generator.Context, t *ty
Package: "go.uber.org/zap",
Name: "SugaredLogger",
}),
"zapDebugLevel": c.Universe.Type(types.Name{
Package: "go.uber.org/zapcore",
Name: "DebugLevel",
}),
"setsNewString": c.Universe.Function(types.Name{
Package: "k8s.io/apimachinery/pkg/util/sets",
Name: "NewString",
@ -520,7 +524,7 @@ func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) erro
// the elected leader is expected to write modifications.
logger.Warn("Saw status changes when we aren't the leader!")
default:
if err = r.updateStatus(ctx, original, resource); err != nil {
if err = r.updateStatus(ctx, logger, original, resource); err != nil {
logger.Warnw("Failed to update resource status", zap.Error(err))
r.Recorder.Eventf(resource, {{.corev1EventTypeWarning|raw}}, "UpdateFailed",
"Failed to update status for %q: %v", resource.Name, err)
@ -559,7 +563,7 @@ func (r *reconcilerImpl) Reconcile(ctx {{.contextContext|raw}}, key string) erro
`
var reconcilerStatusFactory = `
func (r *reconcilerImpl) updateStatus(ctx {{.contextContext|raw}}, existing *{{.type|raw}}, desired *{{.type|raw}}) error {
func (r *reconcilerImpl) updateStatus(ctx {{.contextContext|raw}}, logger *{{.zapSugaredLogger|raw}}, existing *{{.type|raw}}, desired *{{.type|raw}}) error {
existing = existing.DeepCopy()
return {{.reconcilerRetryUpdateConflicts|raw}}(func(attempts int) (err error) {
// The first iteration tries to use the injectionInformer's state, subsequent attempts fetch the latest state via API.
@ -580,8 +584,10 @@ func (r *reconcilerImpl) updateStatus(ctx {{.contextContext|raw}}, existing *{{.
return nil
}
if diff, err := {{.kmpSafeDiff|raw}}(existing.Status, desired.Status); err == nil && diff != "" {
{{.loggingFromContext|raw}}(ctx).Debug("Updating status with: ", diff)
if logger.Desugar().Core().Enabled(zapcore.DebugLevel) {
if diff, err := {{.kmpSafeDiff|raw}}(existing.Status, desired.Status); err == nil && diff != "" {
logger.Debug("Updating status with: ", diff)
}
}
existing.Status = desired.Status

View File

@ -46,6 +46,10 @@ type Options struct {
// Objects that pass the filter (return true) will be reconciled when a new leader is promoted.
// If no filter is specified, all objects will be reconciled.
PromoteFilterFunc func(obj interface{}) bool
// PromoteFunc is called when a reconciler is promoted for the given bucket
// The provided function must not block execution.
PromoteFunc func(bkt reconciler.Bucket)
}
// OptionsFn is a callback method signature that accepts an Impl and returns

View File

@ -0,0 +1,109 @@
/*
Copyright 2023 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package injection
import (
"context"
"errors"
"net/http"
"strconv"
"time"
"knative.dev/pkg/logging"
)
// HealthCheckDefaultPort defines the default port number for health probes
const HealthCheckDefaultPort = 8080
// ServeHealthProbes sets up liveness and readiness probes.
// If user sets no probes explicitly via the context then defaults are added.
func ServeHealthProbes(ctx context.Context, port int) error {
logger := logging.FromContext(ctx)
server := http.Server{ReadHeaderTimeout: time.Minute, Handler: muxWithHandles(ctx), Addr: ":" + strconv.Itoa(port)}
go func() {
<-ctx.Done()
_ = server.Shutdown(ctx)
}()
// start the web server on port and accept requests
logger.Infof("Probes server listening on port %s", port)
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
func muxWithHandles(ctx context.Context) *http.ServeMux {
mux := http.NewServeMux()
readiness := getReadinessHandleOrDefault(ctx)
liveness := getLivenessHandleOrDefault(ctx)
mux.HandleFunc("/readiness", *readiness)
mux.HandleFunc("/health", *liveness)
return mux
}
func newDefaultProbesHandle(sigCtx context.Context) http.HandlerFunc {
logger := logging.FromContext(sigCtx)
return func(w http.ResponseWriter, r *http.Request) {
f := func() error {
select {
// When we get SIGTERM (sigCtx done), let readiness probes start failing.
case <-sigCtx.Done():
logger.Info("Signal context canceled")
return errors.New("received SIGTERM from kubelet")
default:
return nil
}
}
if err := f(); err != nil {
logger.Errorf("Healthcheck failed: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
} else {
w.WriteHeader(http.StatusOK)
}
}
}
type addReadinessKey struct{}
// AddReadiness signals to probe setup logic to add a user provided probe handler
func AddReadiness(ctx context.Context, handlerFunc http.HandlerFunc) context.Context {
return context.WithValue(ctx, addReadinessKey{}, &handlerFunc)
}
func getReadinessHandleOrDefault(ctx context.Context) *http.HandlerFunc {
if ctx.Value(addReadinessKey{}) != nil {
return ctx.Value(addReadinessKey{}).(*http.HandlerFunc)
}
defaultHandle := newDefaultProbesHandle(ctx)
return &defaultHandle
}
type addLivenessKey struct{}
// AddLiveness signals to probe setup logic to add a user provided probe handler
func AddLiveness(ctx context.Context, handlerFunc http.HandlerFunc) context.Context {
return context.WithValue(ctx, addLivenessKey{}, &handlerFunc)
}
func getLivenessHandleOrDefault(ctx context.Context) *http.HandlerFunc {
if ctx.Value(addLivenessKey{}) != nil {
return ctx.Value(addLivenessKey{}).(*http.HandlerFunc)
}
defaultHandle := newDefaultProbesHandle(ctx)
return &defaultHandle
}

View File

@ -20,6 +20,7 @@ import (
"net/http"
"strconv"
"sync"
"time"
prom "contrib.go.opencensus.io/exporter/prometheus"
"go.opencensus.io/resource"
@ -82,10 +83,10 @@ func startNewPromSrv(e *prom.Exporter, host string, port int) *http.Server {
if curPromSrv != nil {
curPromSrv.Close()
}
//nolint:gosec
curPromSrv = &http.Server{
Addr: host + ":" + strconv.Itoa(port),
Handler: sm,
Addr: host + ":" + strconv.Itoa(port),
Handler: sm,
ReadHeaderTimeout: time.Minute, //https://medium.com/a-journey-with-go/go-understand-and-mitigate-slowloris-attack-711c1b1403f6
}
return curPromSrv
}

View File

@ -21,6 +21,7 @@ import (
"crypto/tls"
"net"
"net/http"
"time"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
@ -28,10 +29,10 @@ import (
// NewServer returns a new HTTP Server with HTTP2 handler.
func NewServer(addr string, h http.Handler) *http.Server {
//nolint:gosec
h1s := &http.Server{
Addr: addr,
Handler: h2c.NewHandler(h, &http2.Server{}),
Addr: addr,
Handler: h2c.NewHandler(h, &http2.Server{}),
ReadHeaderTimeout: time.Minute, //https://medium.com/a-journey-with-go/go-understand-and-mitigate-slowloris-attack-711c1b1403f6
}
return h1s

View File

@ -22,6 +22,7 @@ import (
"net/http/pprof"
"os"
"strconv"
"time"
"go.uber.org/atomic"
"go.uber.org/zap"
@ -110,9 +111,9 @@ func NewServer(handler http.Handler) *http.Server {
port = strconv.Itoa(ProfilingPort)
}
//nolint:gosec
return &http.Server{
Addr: ":" + port,
Handler: handler,
Addr: ":" + port,
Handler: handler,
ReadHeaderTimeout: time.Minute, //https://medium.com/a-journey-with-go/go-understand-and-mitigate-slowloris-attack-711c1b1403f6
}
}

View File

@ -265,7 +265,7 @@ func DefaultResponseRetryChecker(resp *Response) (bool, error) {
// logZipkinTrace provides support to log Zipkin Trace for param: spoofResponse
// We only log Zipkin trace for HTTP server errors i.e for HTTP status codes between 500 to 600
func (sc *SpoofingClient) logZipkinTrace(spoofResp *Response) {
if !zipkin.ZipkinTracingEnabled || spoofResp.StatusCode < http.StatusInternalServerError || spoofResp.StatusCode >= 600 {
if !zipkin.IsTracingEnabled() || spoofResp.StatusCode < http.StatusInternalServerError || spoofResp.StatusCode >= 600 {
return
}

View File

@ -31,6 +31,7 @@ import (
"testing"
"time"
"go.uber.org/atomic"
tracingconfig "knative.dev/pkg/tracing/config"
"github.com/openzipkin/zipkin-go/model"
@ -60,9 +61,11 @@ const (
var (
zipkinPortForwardPID int
// ZipkinTracingEnabled variable indicating if zipkin tracing is enabled.
// Deprecated: Use IsTracingEnabled for checking the current state.
ZipkinTracingEnabled = false
tracingEnabled atomic.Bool
// sync.Once variable to ensure we execute zipkin setup only once.
setupOnce sync.Once
@ -70,6 +73,11 @@ var (
teardownOnce sync.Once
)
// IsTracingEnabled indicates if zipkin tracing is enabled.
func IsTracingEnabled() bool {
return tracingEnabled.Load()
}
// SetupZipkinTracingFromConfigTracing setups zipkin tracing like SetupZipkinTracing but retrieving the zipkin configuration
// from config-tracing config map
func SetupZipkinTracingFromConfigTracing(ctx context.Context, kubeClientset kubernetes.Interface, logf logging.FormatLogger, configMapNamespace string) error {
@ -138,6 +146,8 @@ func SetupZipkinTracing(ctx context.Context, kubeClientset kubernetes.Interface,
// Applying AlwaysSample config to ensure we propagate zipkin header for every request made by this client.
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
tracingEnabled.Store(true)
logf("Successfully setup SpoofingClient for Zipkin Tracing")
})
return
@ -166,7 +176,7 @@ func CleanupZipkinTracingSetup(logf logging.FormatLogger) {
// run, SetupZipkinTracing will no longer setup any port forwarding.
setupOnce.Do(func() {})
if !ZipkinTracingEnabled {
if !IsTracingEnabled() {
return
}
@ -175,7 +185,7 @@ func CleanupZipkinTracingSetup(logf logging.FormatLogger) {
return
}
ZipkinTracingEnabled = false
tracingEnabled.Store(false)
})
}

View File

@ -63,10 +63,26 @@ export QUOTA=${QUOTA:-1}
# Receives the latest serving version and searches for the same version with major and minor and searches for the latest patch
function latest_net_istio_version() {
local serving_version=$1
local major_minor
major_minor=$(echo "$serving_version" | cut -d '.' -f 1,2)
local major_minor=$(echo "$serving_version" | cut -d '.' -f 1,2)
curl -L --show-error --silent "https://api.github.com/repos/knative/net-istio/releases" | jq --arg major_minor "$major_minor" -r '[.[].tag_name] | map(select(. | startswith($major_minor))) | sort_by( sub("knative-";"") | sub("v";"") | split(".") | map(tonumber) ) | reverse[0]'
local url="https://api.github.com/repos/knative/net-istio/releases"
local curl_output=$(mktemp)
local curl_flags='-L --show-error --silent'
if [ -n "${GITHUB_TOKEN-}" ]; then
curl $curl_flags -H "Authorization: Bearer $GITHUB_TOKEN" $url > $curl_output
else
curl $curl_flags $url > $curl_output
fi
jq --arg major_minor "$major_minor" -r \
'[.[].tag_name] |
map(select(. | startswith($major_minor))) |
sort_by( sub("knative-";"") |
sub("v";"") |
split(".") |
map(tonumber) ) |
reverse[0]' $curl_output
}
# Latest serving release. If user does not supply this as a flag, the latest

30
vendor/modules.txt vendored
View File

@ -77,7 +77,7 @@ github.com/cpuguy83/go-md2man/v2/md2man
# github.com/davecgh/go-spew v1.1.1
## explicit
github.com/davecgh/go-spew/spew
# github.com/emicklei/go-restful/v3 v3.8.0
# github.com/emicklei/go-restful/v3 v3.9.0
## explicit; go 1.13
github.com/emicklei/go-restful/v3
github.com/emicklei/go-restful/v3/log
@ -152,8 +152,8 @@ github.com/google/go-cmp/cmp/internal/diff
github.com/google/go-cmp/cmp/internal/flags
github.com/google/go-cmp/cmp/internal/function
github.com/google/go-cmp/cmp/internal/value
# github.com/google/go-containerregistry v0.11.0
## explicit; go 1.17
# github.com/google/go-containerregistry v0.13.0
## explicit; go 1.18
github.com/google/go-containerregistry/pkg/name
# github.com/google/gofuzz v1.2.0
## explicit; go 1.12
@ -371,11 +371,11 @@ go.uber.org/zap/internal/bufferpool
go.uber.org/zap/internal/color
go.uber.org/zap/internal/exit
go.uber.org/zap/zapcore
# golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0
# golang.org/x/crypto v0.1.0
## explicit; go 1.17
golang.org/x/crypto/pkcs12
golang.org/x/crypto/pkcs12/internal/rc2
# golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
# golang.org/x/mod v0.6.0
## explicit; go 1.17
golang.org/x/mod/internal/lazyregexp
golang.org/x/mod/module
@ -391,7 +391,7 @@ golang.org/x/net/http2/hpack
golang.org/x/net/idna
golang.org/x/net/internal/timeseries
golang.org/x/net/trace
# golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
# golang.org/x/oauth2 v0.1.0
## explicit; go 1.17
golang.org/x/oauth2
golang.org/x/oauth2/authhandler
@ -400,7 +400,7 @@ golang.org/x/oauth2/google/internal/externalaccount
golang.org/x/oauth2/internal
golang.org/x/oauth2/jws
golang.org/x/oauth2/jwt
# golang.org/x/sync v0.0.0-20220907140024-f12130a52804
# golang.org/x/sync v0.1.0
## explicit
golang.org/x/sync/errgroup
golang.org/x/sync/semaphore
@ -432,7 +432,7 @@ golang.org/x/text/secure/bidirule
golang.org/x/text/transform
golang.org/x/text/unicode/bidi
golang.org/x/text/unicode/norm
# golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45
# golang.org/x/time v0.1.0
## explicit
golang.org/x/time/rate
# golang.org/x/tools v0.1.12
@ -888,7 +888,7 @@ k8s.io/klog/v2/internal/clock
k8s.io/klog/v2/internal/dbg
k8s.io/klog/v2/internal/serialize
k8s.io/klog/v2/internal/severity
# k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1
# k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280
## explicit; go 1.18
k8s.io/kube-openapi/cmd/openapi-gen/args
k8s.io/kube-openapi/pkg/builder3/util
@ -896,7 +896,9 @@ k8s.io/kube-openapi/pkg/common
k8s.io/kube-openapi/pkg/generators
k8s.io/kube-openapi/pkg/generators/rules
k8s.io/kube-openapi/pkg/handler3
k8s.io/kube-openapi/pkg/internal
k8s.io/kube-openapi/pkg/internal/handler
k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json
k8s.io/kube-openapi/pkg/openapiconv
k8s.io/kube-openapi/pkg/schemaconv
k8s.io/kube-openapi/pkg/schemamutation
@ -917,7 +919,7 @@ k8s.io/utils/net
k8s.io/utils/pointer
k8s.io/utils/strings/slices
k8s.io/utils/trace
# knative.dev/eventing v0.36.0
# knative.dev/eventing v0.36.1-0.20230306130433-1ff36e1b656d
## explicit; go 1.18
knative.dev/eventing/pkg/apis/config
knative.dev/eventing/pkg/apis/duck
@ -945,10 +947,10 @@ knative.dev/eventing/pkg/client/clientset/versioned/typed/sources/v1
knative.dev/eventing/pkg/client/clientset/versioned/typed/sources/v1/fake
knative.dev/eventing/pkg/client/clientset/versioned/typed/sources/v1beta2
knative.dev/eventing/pkg/client/clientset/versioned/typed/sources/v1beta2/fake
# knative.dev/hack v0.0.0-20230113013652-c7cfcb062de9
# knative.dev/hack v0.0.0-20230228173453-3de51aff69a3
## explicit; go 1.18
knative.dev/hack
# knative.dev/networking v0.0.0-20230123233838-db2bcbea2560
# knative.dev/networking v0.0.0-20230301131055-c692e9e6afe1
## explicit; go 1.18
knative.dev/networking/pkg
knative.dev/networking/pkg/apis/networking
@ -963,7 +965,7 @@ knative.dev/networking/pkg/http/probe
knative.dev/networking/pkg/http/proxy
knative.dev/networking/pkg/http/stats
knative.dev/networking/pkg/k8s
# knative.dev/pkg v0.0.0-20230117181655-247510c00e9d
# knative.dev/pkg v0.0.0-20230306194819-b77a78c6c0ad
## explicit; go 1.18
knative.dev/pkg/apis
knative.dev/pkg/apis/duck
@ -1013,7 +1015,7 @@ knative.dev/pkg/tracing/config
knative.dev/pkg/tracing/propagation
knative.dev/pkg/tracing/propagation/tracecontextb3
knative.dev/pkg/tracker
# knative.dev/serving v0.36.0
# knative.dev/serving v0.36.1-0.20230306232326-587f58791d1b
## explicit; go 1.18
knative.dev/serving/pkg/apis/autoscaling
knative.dev/serving/pkg/apis/autoscaling/v1alpha1