Add impl for uvip
Kubernetes-commit: cd5f3d9f9d5ae3153206178e6114d573dc24ad73
This commit is contained in:
Richa Banker
Kubernetes Publisher
parent
f100f84ac4
commit
8a38d429d8
22
go.mod
22
go.mod
|
|
@ -41,12 +41,12 @@ require (
|
|||
google.golang.org/protobuf v1.30.0
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
gopkg.in/square/go-jose.v2 v2.6.0
|
||||
k8s.io/api v0.0.0-20230718170521-0c7a73e2a952
|
||||
k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966
|
||||
k8s.io/client-go v0.0.0-20230718174459-8d42d155509f
|
||||
k8s.io/component-base v0.0.0-20230718060631-a69210f80f53
|
||||
k8s.io/api v0.0.0
|
||||
k8s.io/apimachinery v0.0.0
|
||||
k8s.io/client-go v0.0.0
|
||||
k8s.io/component-base v0.0.0
|
||||
k8s.io/klog/v2 v2.100.1
|
||||
k8s.io/kms v0.0.0-20230718060921-6359f9b018b9
|
||||
k8s.io/kms v0.0.0
|
||||
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9
|
||||
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
|
||||
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2
|
||||
|
|
@ -89,6 +89,7 @@ require (
|
|||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/pquerna/cachecontrol v0.1.0 // indirect
|
||||
|
|
@ -125,9 +126,10 @@ require (
|
|||
)
|
||||
|
||||
replace (
|
||||
k8s.io/api => k8s.io/api v0.0.0-20230718170521-0c7a73e2a952
|
||||
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966
|
||||
k8s.io/client-go => k8s.io/client-go v0.0.0-20230718174459-8d42d155509f
|
||||
k8s.io/component-base => k8s.io/component-base v0.0.0-20230718060631-a69210f80f53
|
||||
k8s.io/kms => k8s.io/kms v0.0.0-20230718060921-6359f9b018b9
|
||||
k8s.io/api => ../api
|
||||
k8s.io/apimachinery => ../apimachinery
|
||||
k8s.io/apiserver => ../apiserver
|
||||
k8s.io/client-go => ../client-go
|
||||
k8s.io/component-base => ../component-base
|
||||
k8s.io/kms => ../kms
|
||||
)
|
||||
|
|
|
|||
164
go.sum
164
go.sum
|
|
@ -14,34 +14,156 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ
|
|||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
|
||||
cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
|
||||
cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E=
|
||||
cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ=
|
||||
cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw=
|
||||
cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE=
|
||||
cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8=
|
||||
cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8=
|
||||
cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc=
|
||||
cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E=
|
||||
cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k=
|
||||
cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08=
|
||||
cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU=
|
||||
cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss=
|
||||
cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g=
|
||||
cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU=
|
||||
cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc=
|
||||
cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q=
|
||||
cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8=
|
||||
cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU=
|
||||
cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s=
|
||||
cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA=
|
||||
cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs=
|
||||
cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ=
|
||||
cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w=
|
||||
cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA=
|
||||
cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s=
|
||||
cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8=
|
||||
cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE=
|
||||
cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE=
|
||||
cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8=
|
||||
cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM=
|
||||
cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs=
|
||||
cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4=
|
||||
cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c=
|
||||
cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww=
|
||||
cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ=
|
||||
cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE=
|
||||
cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4=
|
||||
cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs=
|
||||
cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE=
|
||||
cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY=
|
||||
cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU=
|
||||
cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M=
|
||||
cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY=
|
||||
cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg=
|
||||
cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE=
|
||||
cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c=
|
||||
cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0=
|
||||
cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg=
|
||||
cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw=
|
||||
cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw=
|
||||
cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y=
|
||||
cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo=
|
||||
cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
|
||||
cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74=
|
||||
cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4=
|
||||
cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE=
|
||||
cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI=
|
||||
cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY=
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA=
|
||||
cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY=
|
||||
cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I=
|
||||
cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM=
|
||||
cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo=
|
||||
cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw=
|
||||
cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM=
|
||||
cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY=
|
||||
cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU=
|
||||
cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ=
|
||||
cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI=
|
||||
cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ=
|
||||
cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc=
|
||||
cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw=
|
||||
cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs=
|
||||
cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk=
|
||||
cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc=
|
||||
cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4=
|
||||
cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM=
|
||||
cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c=
|
||||
cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac=
|
||||
cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ=
|
||||
cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ=
|
||||
cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI=
|
||||
cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA=
|
||||
cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14=
|
||||
cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg=
|
||||
cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc=
|
||||
cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
|
||||
cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0=
|
||||
cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag=
|
||||
cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s=
|
||||
cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A=
|
||||
cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M=
|
||||
cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw=
|
||||
cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c=
|
||||
cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc=
|
||||
cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM=
|
||||
cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk=
|
||||
cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
|
||||
cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ=
|
||||
cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU=
|
||||
cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0=
|
||||
cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY=
|
||||
cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY=
|
||||
cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes=
|
||||
cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg=
|
||||
cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng=
|
||||
cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
|
||||
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
|
||||
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
|
|
@ -53,6 +175,7 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y
|
|||
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
|
||||
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
|
|
@ -64,11 +187,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
|||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
|
||||
github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
|
||||
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
|
||||
github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk=
|
||||
github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
|
||||
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
|
||||
|
|
@ -77,6 +203,7 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
|
|||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -90,7 +217,9 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
|
|||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
|
||||
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
|
||||
github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
|
||||
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
|
||||
|
|
@ -102,7 +231,9 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
|
|||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
|
|
@ -110,6 +241,7 @@ github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
|
|||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
|
||||
github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
|
||||
github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
|
||||
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
|
||||
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
|
||||
|
|
@ -118,6 +250,7 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/
|
|||
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
|
|
@ -188,6 +321,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
|
|||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
|
|
@ -196,6 +330,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
|
|||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
|
||||
|
|
@ -215,10 +350,12 @@ github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9q
|
|||
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
|
|
@ -226,6 +363,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
|
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
|
|
@ -234,6 +372,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
|
|||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
|
||||
github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
|
|
@ -241,9 +381,15 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
|||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
||||
github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
|
||||
github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
|
||||
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
|
||||
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -263,6 +409,7 @@ github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH
|
|||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
|
|
@ -294,6 +441,7 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ
|
|||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
|
|
@ -316,6 +464,7 @@ go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI=
|
|||
go.etcd.io/etcd/raft/v3 v3.5.9/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg=
|
||||
go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0=
|
||||
go.etcd.io/etcd/server/v3 v3.5.9/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g=
|
||||
go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
|
|
@ -348,6 +497,7 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
|
|||
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
|
||||
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
|
||||
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
|
|
@ -394,6 +544,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
|
|||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -542,10 +693,12 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
|
|||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
|
||||
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
|
|
@ -668,18 +821,9 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
|
|||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
k8s.io/api v0.0.0-20230718170521-0c7a73e2a952 h1:eAohA3wxOcczTsR2at6/LxdV2IQ+BQ3kRwo8PMs7nmk=
|
||||
k8s.io/api v0.0.0-20230718170521-0c7a73e2a952/go.mod h1:DlNtgYu2BX/uM/IkiPaFRqDBVazxK5ggJs19ERbMLZ4=
|
||||
k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966 h1:IYMQQBHbPi6jyorsuYWXB3+kJFH/MxVyLvKJH4nITrg=
|
||||
k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966/go.mod h1:xhQIsaL3hXneGluH+0pzF7kr+VYuLS/VcYJxF1xQf+g=
|
||||
k8s.io/client-go v0.0.0-20230718174459-8d42d155509f h1:3Fc11xFTriyNvnOF3LclaZ5hZhwlN/JT8UnFVp5F6wg=
|
||||
k8s.io/client-go v0.0.0-20230718174459-8d42d155509f/go.mod h1:EKDvjdvs2uH7b2fG8MAqJs37dt7E9oAJJol939kgL5k=
|
||||
k8s.io/component-base v0.0.0-20230718060631-a69210f80f53 h1:YwgpScqW4EwkVxselbAyOjsNm5lxqJRBhzp0acQhGyo=
|
||||
k8s.io/component-base v0.0.0-20230718060631-a69210f80f53/go.mod h1:CWOpOwNtaa1/FdU2/DYyPZgGW/VWTWzNb6uXSwoCA5s=
|
||||
k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
|
||||
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
|
||||
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
|
||||
k8s.io/kms v0.0.0-20230718060921-6359f9b018b9 h1:PklkOaNcMO6G8YMj82VR4+fJuu2Ds4RJ1fKqFsn/cvM=
|
||||
k8s.io/kms v0.0.0-20230718060921-6359f9b018b9/go.mod h1:pa8cdjvp+towzePmbH/bxaqmY2rTNWr3XhOuCug8hy4=
|
||||
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
|
||||
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
|
||||
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,364 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package reconcilers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
kruntime "k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
apirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/apiserver/pkg/registry/rest"
|
||||
"k8s.io/apiserver/pkg/storage"
|
||||
"k8s.io/apiserver/pkg/storage/storagebackend"
|
||||
storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
|
||||
)
|
||||
|
||||
const (
|
||||
APIServerIdentityLabel = "apiserverIdentity"
|
||||
)
|
||||
|
||||
type PeerAdvertiseAddress struct {
|
||||
PeerAdvertiseIP string
|
||||
PeerAdvertisePort string
|
||||
}
|
||||
|
||||
type peerEndpointLeases struct {
|
||||
storage storage.Interface
|
||||
destroyFn func()
|
||||
baseKey string
|
||||
leaseTime time.Duration
|
||||
}
|
||||
|
||||
type PeerEndpointLeaseReconciler interface {
|
||||
// GetEndpoint retrieves the endpoint for a given apiserverId
|
||||
GetEndpoint(serverId string) (string, error)
|
||||
// UpdateLease updates the ip and port of peer servers
|
||||
UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error
|
||||
// RemoveEndpoints removes this apiserver's peer endpoint lease.
|
||||
RemoveLease(serverId string) error
|
||||
// Destroy cleans up everything on shutdown.
|
||||
Destroy()
|
||||
// StopReconciling turns any later ReconcileEndpoints call into a noop.
|
||||
StopReconciling()
|
||||
}
|
||||
|
||||
type peerEndpointLeaseReconciler struct {
|
||||
serverLeases *peerEndpointLeases
|
||||
stopReconcilingCalled atomic.Bool
|
||||
}
|
||||
|
||||
// NewPeerEndpointLeaseReconciler creates a new peer endpoint lease reconciler
|
||||
func NewPeerEndpointLeaseReconciler(config *storagebackend.ConfigForResource, baseKey string, leaseTime time.Duration) (PeerEndpointLeaseReconciler, error) {
|
||||
leaseStorage, destroyFn, err := storagefactory.Create(*config, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating storage factory: %v", err)
|
||||
}
|
||||
var once sync.Once
|
||||
return &peerEndpointLeaseReconciler{
|
||||
serverLeases: &peerEndpointLeases{
|
||||
storage: leaseStorage,
|
||||
destroyFn: func() { once.Do(destroyFn) },
|
||||
baseKey: baseKey,
|
||||
leaseTime: leaseTime,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PeerEndpointController is the controller manager for updating the peer endpoint leases.
|
||||
// This provides a separate independent reconciliation loop for peer endpoint leases
|
||||
// which ensures that the peer kube-apiservers are fetching the updated endpoint info for a given apiserver
|
||||
// in the case when the peer wants to proxy the request to the given apiserver because it can not serve the
|
||||
// request itself due to version mismatch.
|
||||
type PeerEndpointLeaseController struct {
|
||||
reconciler PeerEndpointLeaseReconciler
|
||||
endpointInterval time.Duration
|
||||
serverId string
|
||||
// peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to
|
||||
// route request to this apiserver in case of a version skew.
|
||||
peeraddress string
|
||||
|
||||
client kubernetes.Interface
|
||||
|
||||
lock sync.Mutex
|
||||
stopCh chan struct{} // closed by Stop()
|
||||
}
|
||||
|
||||
func New(serverId string, peeraddress string,
|
||||
reconciler PeerEndpointLeaseReconciler, endpointInterval time.Duration, client kubernetes.Interface) *PeerEndpointLeaseController {
|
||||
return &PeerEndpointLeaseController{
|
||||
reconciler: reconciler,
|
||||
serverId: serverId,
|
||||
// peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to
|
||||
// route request to this apiserver in case of a version skew.
|
||||
peeraddress: peeraddress,
|
||||
endpointInterval: endpointInterval,
|
||||
client: client,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the peer endpoint lease reconciler loop that must exist for bootstrapping
|
||||
// a cluster.
|
||||
func (c *PeerEndpointLeaseController) Start(stopCh <-chan struct{}) {
|
||||
localStopCh := make(chan struct{})
|
||||
go func() {
|
||||
defer close(localStopCh)
|
||||
select {
|
||||
case <-stopCh: // from Start
|
||||
case <-c.stopCh: // from Stop
|
||||
}
|
||||
}()
|
||||
go c.Run(localStopCh)
|
||||
}
|
||||
|
||||
// RunPeerEndpointReconciler periodically updates the peer endpoint leases
|
||||
func (c *PeerEndpointLeaseController) Run(stopCh <-chan struct{}) {
|
||||
// wait until process is ready
|
||||
wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
|
||||
var code int
|
||||
c.client.CoreV1().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&code)
|
||||
return code == http.StatusOK, nil
|
||||
}, stopCh)
|
||||
|
||||
wait.NonSlidingUntil(func() {
|
||||
if err := c.UpdatePeerEndpointLeases(); err != nil {
|
||||
runtime.HandleError(fmt.Errorf("unable to update peer endpoint leases: %v", err))
|
||||
}
|
||||
}, c.endpointInterval, stopCh)
|
||||
}
|
||||
|
||||
// Stop cleans up this apiserver's peer endpoint leases.
|
||||
func (c *PeerEndpointLeaseController) Stop() {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
select {
|
||||
case <-c.stopCh:
|
||||
return // only close once
|
||||
default:
|
||||
close(c.stopCh)
|
||||
}
|
||||
finishedReconciling := make(chan struct{})
|
||||
go func() {
|
||||
defer close(finishedReconciling)
|
||||
klog.Infof("Shutting down peer endpoint lease reconciler")
|
||||
// stop reconciliation
|
||||
c.reconciler.StopReconciling()
|
||||
|
||||
// Ensure that there will be no race condition with the ReconcileEndpointLeases.
|
||||
if err := c.reconciler.RemoveLease(c.serverId); err != nil {
|
||||
klog.Errorf("Unable to remove peer endpoint leases: %v", err)
|
||||
}
|
||||
c.reconciler.Destroy()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-finishedReconciling:
|
||||
// done
|
||||
case <-time.After(2 * c.endpointInterval):
|
||||
// don't block server shutdown forever if we can't reach etcd to remove ourselves
|
||||
klog.Warning("peer_endpoint_controller's RemoveEndpoints() timed out")
|
||||
}
|
||||
}
|
||||
|
||||
// UpdatePeerEndpointLeases attempts to update the peer endpoint leases.
|
||||
func (c *PeerEndpointLeaseController) UpdatePeerEndpointLeases() error {
|
||||
host, port, err := net.SplitHostPort(c.peeraddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
endpointPorts := createEndpointPortSpec(p, "https")
|
||||
|
||||
// Ensure that there will be no race condition with the RemoveEndpointLeases.
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
// Refresh the TTL on our key, independently of whether any error or
|
||||
// update conflict happens below. This makes sure that at least some of
|
||||
// the servers will add our endpoint lease.
|
||||
if err := c.reconciler.UpdateLease(c.serverId, host, endpointPorts); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateLease resets the TTL on a server IP in storage
|
||||
// UpdateLease will create a new key if it doesn't exist.
|
||||
// We use the first element in endpointPorts as a part of the lease's base key
|
||||
// This is done to support out tests that simulate 2 apiservers running on the same ip but
|
||||
// different ports
|
||||
|
||||
// It will also do the following if UnknownVersionInteroperabilityProxy feature is enabled
|
||||
// 1. store the apiserverId as a label
|
||||
// 2. store the values passed to --peer-advertise-ip and --peer-advertise-port flags to kube-apiserver as an annotation
|
||||
// with value of format <ip:port>
|
||||
func (r *peerEndpointLeaseReconciler) UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error {
|
||||
// reconcile endpoints only if apiserver was not shutdown
|
||||
if r.stopReconcilingCalled.Load() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// we use the serverID as the key to avoid using the server IP, port as the key.
|
||||
// note: this means that this lease doesn't enforce mutual exclusion of ip/port usage between apiserver.
|
||||
key := path.Join(r.serverLeases.baseKey, serverId)
|
||||
return r.serverLeases.storage.GuaranteedUpdate(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, true, nil, func(input kruntime.Object, respMeta storage.ResponseMeta) (kruntime.Object, *uint64, error) {
|
||||
existing := input.(*corev1.Endpoints)
|
||||
existing.Subsets = []corev1.EndpointSubset{
|
||||
{
|
||||
Addresses: []corev1.EndpointAddress{{IP: ip}},
|
||||
Ports: endpointPorts,
|
||||
},
|
||||
}
|
||||
|
||||
// store this server's identity (serverId) as a label. This will be used by
|
||||
// peers to find the IP of this server when the peer can not serve a request
|
||||
// due to version skew.
|
||||
if existing.Labels == nil {
|
||||
existing.Labels = map[string]string{}
|
||||
}
|
||||
existing.Labels[APIServerIdentityLabel] = serverId
|
||||
|
||||
// leaseTime needs to be in seconds
|
||||
leaseTime := uint64(r.serverLeases.leaseTime / time.Second)
|
||||
|
||||
// NB: GuaranteedUpdate does not perform the store operation unless
|
||||
// something changed between load and store (not including resource
|
||||
// version), meaning we can't refresh the TTL without actually
|
||||
// changing a field.
|
||||
existing.Generation++
|
||||
|
||||
klog.V(6).Infof("Resetting TTL on server IP %q listed in storage to %v", ip, leaseTime)
|
||||
return existing, &leaseTime, nil
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// ListLeases retrieves a list of the current server IPs from storage
|
||||
func (r *peerEndpointLeaseReconciler) ListLeases() ([]string, error) {
|
||||
storageOpts := storage.ListOptions{
|
||||
ResourceVersion: "0",
|
||||
ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan,
|
||||
Predicate: storage.Everything,
|
||||
Recursive: true,
|
||||
}
|
||||
ipInfoList, err := r.getIpInfoList(storageOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ipList := make([]string, 0, len(ipInfoList.Items))
|
||||
for _, ip := range ipInfoList.Items {
|
||||
if len(ip.Subsets) > 0 && len(ip.Subsets[0].Addresses) > 0 && len(ip.Subsets[0].Addresses[0].IP) > 0 {
|
||||
ipList = append(ipList, ip.Subsets[0].Addresses[0].IP)
|
||||
}
|
||||
}
|
||||
klog.V(6).Infof("Current server IPs listed in storage are %v", ipList)
|
||||
return ipList, nil
|
||||
}
|
||||
|
||||
// GetLease retrieves the server IP and port for a specific server id
|
||||
func (r *peerEndpointLeaseReconciler) GetLease(serverId string) (string, error) {
|
||||
var fullAddr string
|
||||
if serverId == "" {
|
||||
return "", fmt.Errorf("error getting endpoint for serverId: empty serverId")
|
||||
}
|
||||
storageOpts := storage.ListOptions{
|
||||
ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan,
|
||||
Predicate: storage.Everything,
|
||||
Recursive: true,
|
||||
}
|
||||
ipInfoList, err := r.getIpInfoList(storageOpts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, ip := range ipInfoList.Items {
|
||||
if ip.Labels[APIServerIdentityLabel] == serverId {
|
||||
if len(ip.Subsets) > 0 {
|
||||
var ipStr, portStr string
|
||||
if len(ip.Subsets[0].Addresses) > 0 {
|
||||
if len(ip.Subsets[0].Addresses[0].IP) > 0 {
|
||||
ipStr = ip.Subsets[0].Addresses[0].IP
|
||||
}
|
||||
}
|
||||
if len(ip.Subsets[0].Ports) > 0 {
|
||||
portStr = fmt.Sprint(ip.Subsets[0].Ports[0].Port)
|
||||
}
|
||||
fullAddr = net.JoinHostPort(ipStr, portStr)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
klog.V(6).Infof("Fetched this server IP for the specified apiserverId %v, %v", serverId, fullAddr)
|
||||
return fullAddr, nil
|
||||
}
|
||||
|
||||
func (r *peerEndpointLeaseReconciler) StopReconciling() {
|
||||
r.stopReconcilingCalled.Store(true)
|
||||
}
|
||||
|
||||
// RemoveLease removes the lease on a server IP in storage
|
||||
// We use the first element in endpointPorts as a part of the lease's base key
|
||||
// This is done to support out tests that simulate 2 apiservers running on the same ip but
|
||||
// different ports
|
||||
func (r *peerEndpointLeaseReconciler) RemoveLease(serverId string) error {
|
||||
key := path.Join(r.serverLeases.baseKey, serverId)
|
||||
return r.serverLeases.storage.Delete(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, nil, rest.ValidateAllObjectFunc, nil)
|
||||
}
|
||||
|
||||
func (r *peerEndpointLeaseReconciler) Destroy() {
|
||||
r.serverLeases.destroyFn()
|
||||
}
|
||||
|
||||
func (r *peerEndpointLeaseReconciler) GetEndpoint(serverId string) (string, error) {
|
||||
return r.GetLease(serverId)
|
||||
}
|
||||
|
||||
func (r *peerEndpointLeaseReconciler) getIpInfoList(storageOpts storage.ListOptions) (*corev1.EndpointsList, error) {
|
||||
ipInfoList := &corev1.EndpointsList{}
|
||||
if err := r.serverLeases.storage.GetList(apirequest.NewDefaultContext(), r.serverLeases.baseKey, storageOpts, ipInfoList); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ipInfoList, nil
|
||||
}
|
||||
|
||||
// createEndpointPortSpec creates the endpoint ports
|
||||
func createEndpointPortSpec(endpointPort int, endpointPortName string) []corev1.EndpointPort {
|
||||
return []corev1.EndpointPort{{
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
Port: int32(endpointPort),
|
||||
Name: endpointPortName,
|
||||
}}
|
||||
}
|
||||
|
|
@ -0,0 +1,278 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package reconcilers
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/apitesting"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
"k8s.io/apiserver/pkg/storage"
|
||||
etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
|
||||
"k8s.io/apiserver/pkg/storage/storagebackend/factory"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var scheme = runtime.NewScheme()
|
||||
|
||||
metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
|
||||
utilruntime.Must(corev1.AddToScheme(scheme))
|
||||
utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion))
|
||||
|
||||
codecs = serializer.NewCodecFactory(scheme)
|
||||
}
|
||||
|
||||
var codecs serializer.CodecFactory
|
||||
|
||||
type serverInfo struct {
|
||||
existingIP string
|
||||
id string
|
||||
ports []corev1.EndpointPort
|
||||
newIP string
|
||||
removeLease bool
|
||||
expectEndpoint string
|
||||
}
|
||||
|
||||
func NewFakePeerEndpointReconciler(t *testing.T, s storage.Interface) peerEndpointLeaseReconciler {
|
||||
// use the same base key used by the controlplane, but add a random
|
||||
// prefix so we can reuse the etcd instance for subtests independently.
|
||||
base := "/" + uuid.New().String() + "/peerserverleases/"
|
||||
return peerEndpointLeaseReconciler{serverLeases: &peerEndpointLeases{
|
||||
storage: s,
|
||||
destroyFn: func() {},
|
||||
baseKey: base,
|
||||
leaseTime: 1 * time.Minute, // avoid the lease to timeout on tests
|
||||
}}
|
||||
}
|
||||
|
||||
func (f *peerEndpointLeaseReconciler) SetKeys(servers []serverInfo) error {
|
||||
for _, server := range servers {
|
||||
if err := f.UpdateLease(server.id, server.existingIP, server.ports); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPeerEndpointLeaseReconciler(t *testing.T) {
|
||||
// enable feature flags
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)()
|
||||
|
||||
server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
|
||||
t.Cleanup(func() { server.Terminate(t) })
|
||||
|
||||
newFunc := func() runtime.Object { return &corev1.Endpoints{} }
|
||||
sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)
|
||||
|
||||
s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "endpoints"}), newFunc)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating storage: %v", err)
|
||||
}
|
||||
t.Cleanup(dFunc)
|
||||
|
||||
tests := []struct {
|
||||
testName string
|
||||
servers []serverInfo
|
||||
expectLeases []string
|
||||
}{
|
||||
{
|
||||
testName: "existing IP satisfy",
|
||||
servers: []serverInfo{{
|
||||
existingIP: "4.3.2.1",
|
||||
id: "server-1",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
expectEndpoint: "4.3.2.1:8080",
|
||||
}, {
|
||||
existingIP: "1.2.3.4",
|
||||
id: "server-2",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
expectEndpoint: "1.2.3.4:8080",
|
||||
}},
|
||||
expectLeases: []string{"4.3.2.1", "1.2.3.4"},
|
||||
},
|
||||
{
|
||||
testName: "existing IP + new IP = should return the new IP",
|
||||
servers: []serverInfo{{
|
||||
existingIP: "4.3.2.2",
|
||||
id: "server-1",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
newIP: "4.3.2.1",
|
||||
expectEndpoint: "4.3.2.1:8080",
|
||||
}, {
|
||||
existingIP: "1.2.3.4",
|
||||
id: "server-2",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
newIP: "1.1.1.1",
|
||||
expectEndpoint: "1.1.1.1:8080",
|
||||
}},
|
||||
expectLeases: []string{"4.3.2.1", "1.1.1.1"},
|
||||
},
|
||||
{
|
||||
testName: "no existing IP, should return new IP",
|
||||
servers: []serverInfo{{
|
||||
id: "server-1",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
newIP: "1.2.3.4",
|
||||
expectEndpoint: "1.2.3.4:8080",
|
||||
}},
|
||||
expectLeases: []string{"1.2.3.4"},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.testName, func(t *testing.T) {
|
||||
fakeReconciler := NewFakePeerEndpointReconciler(t, s)
|
||||
err := fakeReconciler.SetKeys(test.servers)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error creating keys: %v", err)
|
||||
}
|
||||
|
||||
for _, server := range test.servers {
|
||||
if server.newIP != "" {
|
||||
err = fakeReconciler.UpdateLease(server.id, server.newIP, server.ports)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error reconciling: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
leases, err := fakeReconciler.ListLeases()
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
// sort for comparison
|
||||
sort.Strings(leases)
|
||||
sort.Strings(test.expectLeases)
|
||||
if !reflect.DeepEqual(leases, test.expectLeases) {
|
||||
t.Errorf("expected %v got: %v", test.expectLeases, leases)
|
||||
}
|
||||
|
||||
for _, server := range test.servers {
|
||||
endpoint, err := fakeReconciler.GetLease(server.id)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
if endpoint != server.expectEndpoint {
|
||||
t.Errorf("expected %v got: %v", server.expectEndpoint, endpoint)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerLeaseRemoveEndpoints(t *testing.T) {
|
||||
// enable feature flags
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)()
|
||||
|
||||
server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
|
||||
t.Cleanup(func() { server.Terminate(t) })
|
||||
|
||||
newFunc := func() runtime.Object { return &corev1.Endpoints{} }
|
||||
sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)
|
||||
|
||||
s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), newFunc)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating storage: %v", err)
|
||||
}
|
||||
t.Cleanup(dFunc)
|
||||
|
||||
stopTests := []struct {
|
||||
testName string
|
||||
servers []serverInfo
|
||||
expectLeases []string
|
||||
apiServerStartup bool
|
||||
}{
|
||||
{
|
||||
testName: "successful remove previous endpoints before apiserver starts",
|
||||
servers: []serverInfo{
|
||||
{
|
||||
existingIP: "1.2.3.4",
|
||||
id: "test-server-1",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
removeLease: true,
|
||||
},
|
||||
{
|
||||
existingIP: "2.4.6.8",
|
||||
id: "test-server-2",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
}},
|
||||
expectLeases: []string{"2.4.6.8"},
|
||||
apiServerStartup: true,
|
||||
},
|
||||
{
|
||||
testName: "stop reconciling with new IP not in existing ip list",
|
||||
servers: []serverInfo{{
|
||||
existingIP: "1.2.3.4",
|
||||
newIP: "4.6.8.9",
|
||||
id: "test-server-1",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
},
|
||||
{
|
||||
existingIP: "2.4.6.8",
|
||||
id: "test-server-2",
|
||||
ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
|
||||
removeLease: true,
|
||||
}},
|
||||
expectLeases: []string{"1.2.3.4"},
|
||||
},
|
||||
}
|
||||
for _, test := range stopTests {
|
||||
t.Run(test.testName, func(t *testing.T) {
|
||||
fakeReconciler := NewFakePeerEndpointReconciler(t, s)
|
||||
err := fakeReconciler.SetKeys(test.servers)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error creating keys: %v", err)
|
||||
}
|
||||
if !test.apiServerStartup {
|
||||
fakeReconciler.StopReconciling()
|
||||
}
|
||||
for _, server := range test.servers {
|
||||
if server.removeLease {
|
||||
err = fakeReconciler.RemoveLease(server.id)
|
||||
// if the ip is not on the endpoints, it must return an storage error and stop reconciling
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error reconciling: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
leases, err := fakeReconciler.ListLeases()
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
// sort for comparison
|
||||
sort.Strings(leases)
|
||||
sort.Strings(test.expectLeases)
|
||||
if !reflect.DeepEqual(leases, test.expectLeases) {
|
||||
t.Errorf("expected %v got: %v", test.expectLeases, leases)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -86,6 +86,13 @@ import (
|
|||
_ "k8s.io/apiserver/pkg/apis/apiserver/install"
|
||||
)
|
||||
|
||||
// hostnameFunc is a function to set the hostnameFunc of this apiserver.
|
||||
// To be used for testing purpose only, to simulate scenarios where multiple apiservers
|
||||
// exist. In such cases we want to ensure unique apiserver IDs which are a hash of hostnameFunc.
|
||||
var (
|
||||
hostnameFunc = os.Hostname
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultLegacyAPIPrefix is where the legacy APIs will be located.
|
||||
DefaultLegacyAPIPrefix = "/api"
|
||||
|
|
@ -367,7 +374,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
|
|||
defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
|
||||
var id string
|
||||
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
|
||||
hostname, err := os.Hostname()
|
||||
hostname, err := hostnameFunc()
|
||||
if err != nil {
|
||||
klog.Fatalf("error getting hostname for apiserver identity: %v", err)
|
||||
}
|
||||
|
|
@ -897,7 +904,9 @@ func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c
|
|||
}
|
||||
|
||||
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
|
||||
handler := filterlatency.TrackCompleted(apiHandler)
|
||||
handler := apiHandler
|
||||
|
||||
handler = filterlatency.TrackCompleted(handler)
|
||||
handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
|
||||
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
|
||||
|
||||
|
|
@ -1070,3 +1079,12 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati
|
|||
tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences)
|
||||
authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator)
|
||||
}
|
||||
|
||||
// For testing purpose only
|
||||
func SetHostnameFuncForTests(name string) {
|
||||
hostnameFunc = func() (host string, err error) {
|
||||
host = name
|
||||
err = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
cachermetrics "k8s.io/apiserver/pkg/storage/cacher/metrics"
|
||||
etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
|
||||
flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
|
||||
peerproxymetrics "k8s.io/apiserver/pkg/util/peerproxy/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
|
|
@ -50,4 +51,5 @@ func register() {
|
|||
cachermetrics.Register()
|
||||
etcd3metrics.Register()
|
||||
flowcontrolmetrics.Register()
|
||||
peerproxymetrics.Register()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
const (
|
||||
subsystem = "apiserver"
|
||||
statuscode = "code"
|
||||
)
|
||||
|
||||
var registerMetricsOnce sync.Once
|
||||
|
||||
var (
|
||||
// peerProxiedRequestsTotal counts the number of requests that were proxied to a peer kube-apiserver.
|
||||
peerProxiedRequestsTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Subsystem: subsystem,
|
||||
Name: "rerouted_request_total",
|
||||
Help: "Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{statuscode},
|
||||
)
|
||||
)
|
||||
|
||||
func Register() {
|
||||
registerMetricsOnce.Do(func() {
|
||||
legacyregistry.MustRegister(peerProxiedRequestsTotal)
|
||||
})
|
||||
}
|
||||
|
||||
// IncPeerProxiedRequest increments the # of proxied requests to peer kube-apiserver
|
||||
func IncPeerProxiedRequest(ctx context.Context, status string) {
|
||||
peerProxiedRequestsTotal.WithContext(ctx).WithLabelValues(status).Add(1)
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package peerproxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/reconcilers"
|
||||
"k8s.io/apiserver/pkg/storageversion"
|
||||
kubeinformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
// Interface defines how the Unknown Version Proxy filter interacts with the underlying system.
|
||||
type Interface interface {
|
||||
WrapHandler(handler http.Handler) http.Handler
|
||||
WaitForCacheSync(stopCh <-chan struct{}) error
|
||||
HasFinishedSync() bool
|
||||
}
|
||||
|
||||
// New creates a new instance to implement unknown version proxy
|
||||
func NewPeerProxyHandler(informerFactory kubeinformers.SharedInformerFactory,
|
||||
svm storageversion.Manager,
|
||||
proxyTransport http.RoundTripper,
|
||||
serverId string,
|
||||
reconciler reconcilers.PeerEndpointLeaseReconciler,
|
||||
serializer runtime.NegotiatedSerializer) *peerProxyHandler {
|
||||
h := &peerProxyHandler{
|
||||
name: "PeerProxyHandler",
|
||||
storageversionManager: svm,
|
||||
proxyTransport: proxyTransport,
|
||||
svMap: sync.Map{},
|
||||
serverId: serverId,
|
||||
reconciler: reconciler,
|
||||
serializer: serializer,
|
||||
}
|
||||
svi := informerFactory.Internal().V1alpha1().StorageVersions()
|
||||
h.storageversionInformer = svi.Informer()
|
||||
|
||||
svi.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
h.addSV(obj)
|
||||
},
|
||||
UpdateFunc: func(oldObj, newObj interface{}) {
|
||||
h.updateSV(oldObj, newObj)
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
h.deleteSV(obj)
|
||||
}})
|
||||
return h
|
||||
}
|
||||
|
|
@ -0,0 +1,357 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package peerproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"k8s.io/api/apiserverinternal/v1alpha1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
schema "k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/proxy"
|
||||
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
|
||||
epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
|
||||
apirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/apiserver/pkg/endpoints/responsewriter"
|
||||
"k8s.io/apiserver/pkg/reconcilers"
|
||||
"k8s.io/apiserver/pkg/storageversion"
|
||||
"k8s.io/apiserver/pkg/util/peerproxy/metrics"
|
||||
apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/transport"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
PeerProxiedHeader = "x-kubernetes-peer-proxied"
|
||||
)
|
||||
|
||||
type peerProxyHandler struct {
|
||||
name string
|
||||
// StorageVersion informer used to fetch apiserver ids than can serve a resource
|
||||
storageversionInformer cache.SharedIndexInformer
|
||||
|
||||
// StorageVersion manager used to ensure it has finished updating storageversions before
|
||||
// we start handling external requests
|
||||
storageversionManager storageversion.Manager
|
||||
|
||||
// proxy transport
|
||||
proxyTransport http.RoundTripper
|
||||
|
||||
// identity for this server
|
||||
serverId string
|
||||
|
||||
// reconciler that is used to fetch host port of peer apiserver when proxying request to a peer
|
||||
reconciler reconcilers.PeerEndpointLeaseReconciler
|
||||
|
||||
serializer runtime.NegotiatedSerializer
|
||||
|
||||
// SyncMap for storing an up to date copy of the storageversions and apiservers that can serve them
|
||||
// This map is populated using the StorageVersion informer
|
||||
// This map has key set to GVR and value being another SyncMap
|
||||
// The nested SyncMap has key set to apiserver id and value set to boolean
|
||||
// The nested maps are created to have a "Set" like structure to store unique apiserver ids
|
||||
// for a given GVR
|
||||
svMap sync.Map
|
||||
|
||||
finishedSync atomic.Bool
|
||||
}
|
||||
|
||||
type serviceableByResponse struct {
|
||||
locallyServiceable bool
|
||||
errorFetchingAddressFromLease bool
|
||||
peerEndpoints []string
|
||||
}
|
||||
|
||||
// responder implements rest.Responder for assisting a connector in writing objects or errors.
|
||||
type responder struct {
|
||||
w http.ResponseWriter
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) HasFinishedSync() bool {
|
||||
return h.finishedSync.Load()
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) WaitForCacheSync(stopCh <-chan struct{}) error {
|
||||
|
||||
ok := cache.WaitForNamedCacheSync("unknown-version-proxy", stopCh, h.storageversionInformer.HasSynced, h.storageversionManager.Completed)
|
||||
if !ok {
|
||||
return fmt.Errorf("error while waiting for initial cache sync")
|
||||
}
|
||||
klog.V(3).Infof("setting finishedSync to true")
|
||||
h.finishedSync.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// WrapHandler will fetch the apiservers that can serve the request and either serve it locally
|
||||
// or route it to a peer
|
||||
func (h *peerProxyHandler) WrapHandler(handler http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
requestInfo, ok := apirequest.RequestInfoFrom(ctx)
|
||||
|
||||
if !ok {
|
||||
responsewriters.InternalError(w, r, errors.New("no RequestInfo found in the context"))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow non-resource requests
|
||||
if !requestInfo.IsResourceRequest {
|
||||
klog.V(3).Infof("Not a resource request skipping proxying")
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Request has already been proxied once, it must be served locally
|
||||
if r.Header.Get(PeerProxiedHeader) == "true" {
|
||||
klog.V(3).Infof("Already rerouted once, skipping proxying to peer")
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// StorageVersion Informers and/or StorageVersionManager is not synced yet, pass request to next handler
|
||||
// This will happen for self requests from the kube-apiserver because we have a poststarthook
|
||||
// to ensure that external requests are not served until the StorageVersion Informer and
|
||||
// StorageVersionManager has synced
|
||||
if !h.HasFinishedSync() {
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
gvr := schema.GroupVersionResource{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion, Resource: requestInfo.Resource}
|
||||
if requestInfo.APIGroup == "" {
|
||||
gvr.Group = "core"
|
||||
}
|
||||
|
||||
// find servers that are capable of serving this request
|
||||
serviceableByResp, err := h.findServiceableByServers(gvr, h.serverId, h.reconciler)
|
||||
if err != nil {
|
||||
// this means that resource is an aggregated API or a CR since it wasn't found in SV informer cache, pass as it is
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
// found the gvr locally, pass request to the next handler in local apiserver
|
||||
if serviceableByResp.locallyServiceable {
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
gv := schema.GroupVersion{Group: gvr.Group, Version: gvr.Version}
|
||||
|
||||
if serviceableByResp.errorFetchingAddressFromLease {
|
||||
klog.ErrorS(err, "error fetching ip and port of remote server while proxying")
|
||||
responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable("Error getting ip and port info of the remote server while proxying"), h.serializer, gv, w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// no apiservers were found that could serve the request, pass request to
|
||||
// next handler, that should eventually serve 404
|
||||
|
||||
// TODO: maintain locally serviceable GVRs somewhere so that we dont have to
|
||||
// consult the storageversion-informed map for those
|
||||
if len(serviceableByResp.peerEndpoints) == 0 {
|
||||
klog.Errorf(fmt.Sprintf("GVR %v is not served by anything in this cluster", gvr))
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// otherwise, randomly select an apiserver and proxy request to it
|
||||
rand := rand.Intn(len(serviceableByResp.peerEndpoints))
|
||||
destServerHostPort := serviceableByResp.peerEndpoints[rand]
|
||||
h.proxyRequestToDestinationAPIServer(r, w, destServerHostPort)
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) findServiceableByServers(gvr schema.GroupVersionResource, localAPIServerId string, reconciler reconcilers.PeerEndpointLeaseReconciler) (serviceableByResponse, error) {
|
||||
|
||||
apiserversi, ok := h.svMap.Load(gvr)
|
||||
|
||||
// no value found for the requested gvr in svMap
|
||||
if !ok || apiserversi == nil {
|
||||
return serviceableByResponse{}, fmt.Errorf("no StorageVersions found for the GVR: %v", gvr)
|
||||
}
|
||||
apiservers := apiserversi.(*sync.Map)
|
||||
response := serviceableByResponse{}
|
||||
var peerServerEndpoints []string
|
||||
apiservers.Range(func(key, value interface{}) bool {
|
||||
apiserverKey := key.(string)
|
||||
if apiserverKey == localAPIServerId {
|
||||
response.errorFetchingAddressFromLease = true
|
||||
response.locallyServiceable = true
|
||||
// stop iteration
|
||||
return false
|
||||
}
|
||||
|
||||
hostPort, err := reconciler.GetEndpoint(apiserverKey)
|
||||
if err != nil {
|
||||
response.errorFetchingAddressFromLease = true
|
||||
klog.Errorf("failed to get peer ip from storage lease for server %s", apiserverKey)
|
||||
// continue with iteration
|
||||
return true
|
||||
}
|
||||
// check ip format
|
||||
_, _, err = net.SplitHostPort(hostPort)
|
||||
if err != nil {
|
||||
response.errorFetchingAddressFromLease = true
|
||||
klog.Errorf("invalid address found for server %s", apiserverKey)
|
||||
// continue with iteration
|
||||
return true
|
||||
}
|
||||
peerServerEndpoints = append(peerServerEndpoints, hostPort)
|
||||
// continue with iteration
|
||||
return true
|
||||
})
|
||||
|
||||
response.peerEndpoints = peerServerEndpoints
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) proxyRequestToDestinationAPIServer(req *http.Request, rw http.ResponseWriter, host string) {
|
||||
user, ok := apirequest.UserFrom(req.Context())
|
||||
if !ok {
|
||||
klog.Errorf("failed to get user info from request")
|
||||
return
|
||||
}
|
||||
|
||||
// write a new location based on the existing request pointed at the target service
|
||||
location := &url.URL{}
|
||||
location.Scheme = "https"
|
||||
location.Host = host
|
||||
location.Path = req.URL.Path
|
||||
location.RawQuery = req.URL.Query().Encode()
|
||||
|
||||
newReq, cancelFn := apiserverproxyutil.NewRequestForProxy(location, req)
|
||||
newReq.Header.Add(PeerProxiedHeader, "true")
|
||||
defer cancelFn()
|
||||
|
||||
proxyRoundTripper := transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), h.proxyTransport)
|
||||
|
||||
delegate := &epmetrics.ResponseWriterDelegator{ResponseWriter: rw}
|
||||
w := responsewriter.WrapForHTTP1Or2(delegate)
|
||||
|
||||
handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, false, &responder{w: w, ctx: req.Context()})
|
||||
handler.ServeHTTP(w, newReq)
|
||||
// Increment the count of proxied requests
|
||||
metrics.IncPeerProxiedRequest(req.Context(), strconv.Itoa(delegate.Status()))
|
||||
}
|
||||
|
||||
func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) {
|
||||
klog.Errorf("Error while proxying request to destination apiserver: %v", err)
|
||||
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
// Adds a storageversion object to SVMap
|
||||
func (h *peerProxyHandler) addSV(obj interface{}) {
|
||||
sv, ok := obj.(*v1alpha1.StorageVersion)
|
||||
if !ok {
|
||||
klog.Errorf("Invalid StorageVersion provided to addSV()")
|
||||
return
|
||||
}
|
||||
h.updateSVMap(nil, sv)
|
||||
}
|
||||
|
||||
// Updates the SVMap to delete old storageversion and add new storageversion
|
||||
func (h *peerProxyHandler) updateSV(oldObj interface{}, newObj interface{}) {
|
||||
oldSV, ok := oldObj.(*v1alpha1.StorageVersion)
|
||||
if !ok {
|
||||
klog.Errorf("Invalid StorageVersion provided to updateSV()")
|
||||
return
|
||||
}
|
||||
newSV, ok := newObj.(*v1alpha1.StorageVersion)
|
||||
if !ok {
|
||||
klog.Errorf("Invalid StorageVersion provided to updateSV()")
|
||||
return
|
||||
}
|
||||
h.updateSVMap(oldSV, newSV)
|
||||
}
|
||||
|
||||
// Deletes a storageversion object from SVMap
|
||||
func (h *peerProxyHandler) deleteSV(obj interface{}) {
|
||||
sv, ok := obj.(*v1alpha1.StorageVersion)
|
||||
if !ok {
|
||||
klog.Errorf("Invalid StorageVersion provided to deleteSV()")
|
||||
return
|
||||
}
|
||||
h.updateSVMap(sv, nil)
|
||||
}
|
||||
|
||||
// Delete old storageversion, add new storagversion
|
||||
func (h *peerProxyHandler) updateSVMap(oldSV *v1alpha1.StorageVersion, newSV *v1alpha1.StorageVersion) {
|
||||
if oldSV != nil {
|
||||
// delete old SV entries
|
||||
h.deleteSVFromMap(oldSV)
|
||||
}
|
||||
if newSV != nil {
|
||||
// add new SV entries
|
||||
h.addSVToMap(newSV)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) deleteSVFromMap(sv *v1alpha1.StorageVersion) {
|
||||
// The name of storageversion is <group>.<resource>
|
||||
splitInd := strings.LastIndex(sv.Name, ".")
|
||||
group := sv.Name[:splitInd]
|
||||
resource := sv.Name[splitInd+1:]
|
||||
|
||||
gvr := schema.GroupVersionResource{Group: group, Resource: resource}
|
||||
for _, gr := range sv.Status.StorageVersions {
|
||||
for _, version := range gr.ServedVersions {
|
||||
versionSplit := strings.Split(version, "/")
|
||||
if len(versionSplit) == 2 {
|
||||
version = versionSplit[1]
|
||||
}
|
||||
gvr.Version = version
|
||||
h.svMap.Delete(gvr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) addSVToMap(sv *v1alpha1.StorageVersion) {
|
||||
// The name of storageversion is <group>.<resource>
|
||||
splitInd := strings.LastIndex(sv.Name, ".")
|
||||
group := sv.Name[:splitInd]
|
||||
resource := sv.Name[splitInd+1:]
|
||||
|
||||
gvr := schema.GroupVersionResource{Group: group, Resource: resource}
|
||||
for _, gr := range sv.Status.StorageVersions {
|
||||
for _, version := range gr.ServedVersions {
|
||||
|
||||
// some versions have groups included in them, so get rid of the groups
|
||||
versionSplit := strings.Split(version, "/")
|
||||
if len(versionSplit) == 2 {
|
||||
version = versionSplit[1]
|
||||
}
|
||||
gvr.Version = version
|
||||
apiserversi, _ := h.svMap.LoadOrStore(gvr, &sync.Map{})
|
||||
apiservers := apiserversi.(*sync.Map)
|
||||
apiservers.Store(gr.APIServerID, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,329 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package peerproxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/apitesting"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
apifilters "k8s.io/apiserver/pkg/endpoints/filters"
|
||||
apirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
"k8s.io/apiserver/pkg/reconcilers"
|
||||
etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
|
||||
"k8s.io/apiserver/pkg/storageversion"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/apiserver/pkg/util/peerproxy/metrics"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
"k8s.io/client-go/transport"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
requestTimeout = 30 * time.Second
|
||||
localServerId = "local-apiserver"
|
||||
remoteServerId = "remote-apiserver"
|
||||
)
|
||||
|
||||
type FakeSVMapData struct {
|
||||
gvr schema.GroupVersionResource
|
||||
serverId string
|
||||
}
|
||||
|
||||
type reconciler struct {
|
||||
do bool
|
||||
publicIP string
|
||||
serverId string
|
||||
}
|
||||
|
||||
func TestPeerProxy(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
svdata FakeSVMapData
|
||||
informerFinishedSync bool
|
||||
requestPath string
|
||||
peerproxiedHeader string
|
||||
expectedStatus int
|
||||
metrics []string
|
||||
want string
|
||||
reconcilerConfig reconciler
|
||||
}{
|
||||
{
|
||||
desc: "allow non resource requests",
|
||||
requestPath: "/foo/bar/baz",
|
||||
expectedStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
desc: "allow if already proxied once",
|
||||
requestPath: "/api/bar/baz",
|
||||
expectedStatus: http.StatusOK,
|
||||
peerproxiedHeader: "true",
|
||||
},
|
||||
{
|
||||
desc: "allow if unsynced informers",
|
||||
requestPath: "/api/bar/baz",
|
||||
expectedStatus: http.StatusOK,
|
||||
informerFinishedSync: false,
|
||||
},
|
||||
{
|
||||
desc: "allow if no storage version found",
|
||||
requestPath: "/api/bar/baz",
|
||||
expectedStatus: http.StatusOK,
|
||||
informerFinishedSync: true,
|
||||
},
|
||||
{
|
||||
// since if no server id is found, we pass request to next handler
|
||||
//, and our last handler in local chain is an http ok handler
|
||||
desc: "200 if no serverid found",
|
||||
requestPath: "/api/bar/baz",
|
||||
expectedStatus: http.StatusOK,
|
||||
informerFinishedSync: true,
|
||||
svdata: FakeSVMapData{
|
||||
gvr: schema.GroupVersionResource{
|
||||
Group: "core",
|
||||
Version: "bar",
|
||||
Resource: "baz"},
|
||||
serverId: ""},
|
||||
},
|
||||
{
|
||||
desc: "503 if no endpoint fetched from lease",
|
||||
requestPath: "/api/foo/bar",
|
||||
expectedStatus: http.StatusServiceUnavailable,
|
||||
informerFinishedSync: true,
|
||||
svdata: FakeSVMapData{
|
||||
gvr: schema.GroupVersionResource{
|
||||
Group: "core",
|
||||
Version: "foo",
|
||||
Resource: "bar"},
|
||||
serverId: remoteServerId},
|
||||
},
|
||||
{
|
||||
desc: "200 if locally serviceable",
|
||||
requestPath: "/api/foo/bar",
|
||||
expectedStatus: http.StatusOK,
|
||||
informerFinishedSync: true,
|
||||
svdata: FakeSVMapData{
|
||||
gvr: schema.GroupVersionResource{
|
||||
Group: "core",
|
||||
Version: "foo",
|
||||
Resource: "bar"},
|
||||
serverId: localServerId},
|
||||
},
|
||||
{
|
||||
desc: "503 unreachable peer bind address",
|
||||
requestPath: "/api/foo/bar",
|
||||
expectedStatus: http.StatusServiceUnavailable,
|
||||
informerFinishedSync: true,
|
||||
svdata: FakeSVMapData{
|
||||
gvr: schema.GroupVersionResource{
|
||||
Group: "core",
|
||||
Version: "foo",
|
||||
Resource: "bar"},
|
||||
serverId: remoteServerId},
|
||||
reconcilerConfig: reconciler{
|
||||
do: true,
|
||||
publicIP: "1.2.3.4",
|
||||
serverId: remoteServerId,
|
||||
},
|
||||
metrics: []string{
|
||||
"apiserver_rerouted_request_total",
|
||||
},
|
||||
want: `
|
||||
# HELP apiserver_rerouted_request_total [ALPHA] Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it
|
||||
# TYPE apiserver_rerouted_request_total counter
|
||||
apiserver_rerouted_request_total{code="503"} 1
|
||||
`,
|
||||
},
|
||||
{
|
||||
desc: "503 unreachable peer public address",
|
||||
requestPath: "/api/foo/bar",
|
||||
expectedStatus: http.StatusServiceUnavailable,
|
||||
informerFinishedSync: true,
|
||||
svdata: FakeSVMapData{
|
||||
gvr: schema.GroupVersionResource{
|
||||
Group: "core",
|
||||
Version: "foo",
|
||||
Resource: "bar"},
|
||||
serverId: remoteServerId},
|
||||
reconcilerConfig: reconciler{
|
||||
do: true,
|
||||
publicIP: "1.2.3.4",
|
||||
serverId: remoteServerId,
|
||||
},
|
||||
metrics: []string{
|
||||
"apiserver_rerouted_request_total",
|
||||
},
|
||||
want: `
|
||||
# HELP apiserver_rerouted_request_total [ALPHA] Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it
|
||||
# TYPE apiserver_rerouted_request_total counter
|
||||
apiserver_rerouted_request_total{code="503"} 2
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
metrics.Register()
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.desc, func(t *testing.T) {
|
||||
lastHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("OK"))
|
||||
})
|
||||
reconciler := newFakePeerEndpointReconciler(t)
|
||||
handler := newHandlerChain(t, lastHandler, reconciler, tt.informerFinishedSync, tt.svdata)
|
||||
server, requestGetter := createHTTP2ServerWithClient(handler, requestTimeout*2)
|
||||
defer server.Close()
|
||||
|
||||
if tt.reconcilerConfig.do {
|
||||
// need to enable feature flags first
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)()
|
||||
|
||||
reconciler.UpdateLease(tt.reconcilerConfig.serverId,
|
||||
tt.reconcilerConfig.publicIP,
|
||||
[]corev1.EndpointPort{{Name: "foo",
|
||||
Port: 8080, Protocol: "TCP"}})
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, server.URL+tt.requestPath, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create new http request - %v", err)
|
||||
}
|
||||
req.Header.Set(PeerProxiedHeader, tt.peerproxiedHeader)
|
||||
|
||||
resp, _ := requestGetter(req)
|
||||
|
||||
// compare response
|
||||
assert.Equal(t, tt.expectedStatus, resp.StatusCode)
|
||||
|
||||
// compare metric
|
||||
if tt.want != "" {
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func newFakePeerEndpointReconciler(t *testing.T) reconcilers.PeerEndpointLeaseReconciler {
|
||||
server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
|
||||
t.Cleanup(func() { server.Terminate(t) })
|
||||
scheme := runtime.NewScheme()
|
||||
metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
|
||||
//utilruntime.Must(core.AddToScheme(scheme))
|
||||
utilruntime.Must(corev1.AddToScheme(scheme))
|
||||
utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion))
|
||||
codecs := serializer.NewCodecFactory(scheme)
|
||||
sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)
|
||||
config := *sc.ForResource(schema.GroupResource{Resource: "endpoints"})
|
||||
baseKey := "/" + uuid.New().String() + "/peer-testleases/"
|
||||
leaseTime := 1 * time.Minute
|
||||
reconciler, err := reconcilers.NewPeerEndpointLeaseReconciler(&config, baseKey, leaseTime)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating storage: %v", err)
|
||||
}
|
||||
return reconciler
|
||||
}
|
||||
|
||||
func newHandlerChain(t *testing.T, handler http.Handler, reconciler reconcilers.PeerEndpointLeaseReconciler, informerFinishedSync bool, svdata FakeSVMapData) http.Handler {
|
||||
// Add peerproxy handler
|
||||
s := serializer.NewCodecFactory(runtime.NewScheme()).WithoutConversion()
|
||||
peerProxyHandler, err := newFakePeerProxyHandler(informerFinishedSync, reconciler, svdata, localServerId, s)
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating peer proxy handler: %v", err)
|
||||
}
|
||||
peerProxyHandler.finishedSync.Store(informerFinishedSync)
|
||||
handler = peerProxyHandler.WrapHandler(handler)
|
||||
|
||||
// Add user info
|
||||
handler = withFakeUser(handler)
|
||||
|
||||
// Add requestInfo handler
|
||||
requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
|
||||
handler = apifilters.WithRequestInfo(handler, requestInfoFactory)
|
||||
return handler
|
||||
}
|
||||
|
||||
func newFakePeerProxyHandler(informerFinishedSync bool, reconciler reconcilers.PeerEndpointLeaseReconciler, svdata FakeSVMapData, id string, s runtime.NegotiatedSerializer) (*peerProxyHandler, error) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
informerFactory := informers.NewSharedInformerFactory(clientset, 0)
|
||||
clientConfig := &transport.Config{
|
||||
TLS: transport.TLSConfig{
|
||||
Insecure: false,
|
||||
}}
|
||||
proxyRoundTripper, err := transport.New(clientConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ppI := NewPeerProxyHandler(informerFactory, storageversion.NewDefaultManager(), proxyRoundTripper, id, reconciler, s)
|
||||
if testDataExists(svdata.gvr) {
|
||||
ppI.addToStorageVersionMap(svdata.gvr, svdata.serverId)
|
||||
}
|
||||
return ppI, nil
|
||||
}
|
||||
|
||||
func (h *peerProxyHandler) addToStorageVersionMap(gvr schema.GroupVersionResource, serverId string) {
|
||||
apiserversi, _ := h.svMap.LoadOrStore(gvr, &sync.Map{})
|
||||
apiservers := apiserversi.(*sync.Map)
|
||||
if serverId != "" {
|
||||
apiservers.Store(serverId, true)
|
||||
}
|
||||
}
|
||||
|
||||
func testDataExists(gvr schema.GroupVersionResource) bool {
|
||||
return gvr.Group != "" && gvr.Version != "" && gvr.Resource != ""
|
||||
}
|
||||
|
||||
func withFakeUser(handler http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
|
||||
Groups: r.Header["Groups"],
|
||||
}))
|
||||
handler.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// returns a started http2 server, with a client function to send request to the server.
|
||||
func createHTTP2ServerWithClient(handler http.Handler, clientTimeout time.Duration) (*httptest.Server, func(req *http.Request) (*http.Response, error)) {
|
||||
server := httptest.NewUnstartedServer(handler)
|
||||
server.EnableHTTP2 = true
|
||||
server.StartTLS()
|
||||
cli := server.Client()
|
||||
cli.Timeout = clientTimeout
|
||||
return server, func(req *http.Request) (*http.Response, error) {
|
||||
return cli.Do(req)
|
||||
}
|
||||
}
|
||||
|
|
@ -17,17 +17,30 @@ limitations under the License.
|
|||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
utilnet "k8s.io/apimachinery/pkg/util/net"
|
||||
auditinternal "k8s.io/apiserver/pkg/apis/audit"
|
||||
"k8s.io/apiserver/pkg/audit"
|
||||
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||
listersv1 "k8s.io/client-go/listers/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
// taken from https://github.com/kubernetes/kubernetes/blob/release-1.27/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go#L47
|
||||
aggregatedDiscoveryTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// findServicePort finds the service port by name or numerically.
|
||||
func findServicePort(svc *v1.Service, port int32) (*v1.ServicePort, error) {
|
||||
for _, svcPort := range svc.Spec.Ports {
|
||||
|
|
@ -117,3 +130,34 @@ func ResolveCluster(services listersv1.ServiceLister, namespace, id string, port
|
|||
return nil, fmt.Errorf("unsupported service type %q", svc.Spec.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// NewRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests
|
||||
func NewRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) {
|
||||
newCtx := req.Context()
|
||||
cancelFn := func() {}
|
||||
|
||||
if requestInfo, ok := genericapirequest.RequestInfoFrom(req.Context()); ok {
|
||||
// trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three
|
||||
// segments that we are going to proxy, we have a discovery request.
|
||||
if !requestInfo.IsResourceRequest && len(strings.Split(strings.Trim(requestInfo.Path, "/"), "/")) == 3 {
|
||||
// discovery requests are used by kubectl and others to determine which resources a server has. This is a cheap call that
|
||||
// should be fast for every aggregated apiserver. Latency for aggregation is expected to be low (as for all extensions)
|
||||
// so forcing a short timeout here helps responsiveness of all clients.
|
||||
newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// WithContext creates a shallow clone of the request with the same context.
|
||||
newReq := req.WithContext(newCtx)
|
||||
newReq.Header = utilnet.CloneHeader(req.Header)
|
||||
newReq.URL = location
|
||||
newReq.Host = location.Host
|
||||
|
||||
// If the original request has an audit ID, let's make sure we propagate this
|
||||
// to the aggregated server.
|
||||
if auditID, found := audit.AuditIDFrom(req.Context()); found {
|
||||
newReq.Header.Set(auditinternal.HeaderAuditID, string(auditID))
|
||||
}
|
||||
|
||||
return newReq, cancelFn
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue