diff --git a/go.mod b/go.mod index 5e0408bc3..e1ca4fde1 100644 --- a/go.mod +++ b/go.mod @@ -41,12 +41,12 @@ require ( google.golang.org/protobuf v1.30.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/square/go-jose.v2 v2.6.0 - k8s.io/api v0.0.0-20230718170521-0c7a73e2a952 - k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966 - k8s.io/client-go v0.0.0-20230718174459-8d42d155509f - k8s.io/component-base v0.0.0-20230718060631-a69210f80f53 + k8s.io/api v0.0.0 + k8s.io/apimachinery v0.0.0 + k8s.io/client-go v0.0.0 + k8s.io/component-base v0.0.0 k8s.io/klog/v2 v2.100.1 - k8s.io/kms v0.0.0-20230718060921-6359f9b018b9 + k8s.io/kms v0.0.0 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 @@ -89,6 +89,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect @@ -125,9 +126,10 @@ require ( ) replace ( - k8s.io/api => k8s.io/api v0.0.0-20230718170521-0c7a73e2a952 - k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966 - k8s.io/client-go => k8s.io/client-go v0.0.0-20230718174459-8d42d155509f - k8s.io/component-base => k8s.io/component-base v0.0.0-20230718060631-a69210f80f53 - k8s.io/kms => k8s.io/kms v0.0.0-20230718060921-6359f9b018b9 + k8s.io/api => ../api + k8s.io/apimachinery => ../apimachinery + k8s.io/apiserver => ../apiserver + k8s.io/client-go => ../client-go + k8s.io/component-base => ../component-base + k8s.io/kms => ../kms ) diff --git a/go.sum b/go.sum index 737f1f126..732caf1a5 100644 --- a/go.sum +++ b/go.sum @@ -14,34 +14,156 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -53,6 +175,7 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -64,11 +187,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -77,6 +203,7 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -90,7 +217,9 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= @@ -102,7 +231,9 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -110,6 +241,7 @@ github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -118,6 +250,7 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -188,6 +321,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -196,6 +330,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -215,10 +350,12 @@ github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9q github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -226,6 +363,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -234,6 +372,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -241,9 +381,15 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -263,6 +409,7 @@ github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -294,6 +441,7 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -316,6 +464,7 @@ go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI= go.etcd.io/etcd/raft/v3 v3.5.9/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg= go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0= go.etcd.io/etcd/server/v3 v3.5.9/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= +go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -348,6 +497,7 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -394,6 +544,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -542,10 +693,12 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -668,18 +821,9 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.0.0-20230718170521-0c7a73e2a952 h1:eAohA3wxOcczTsR2at6/LxdV2IQ+BQ3kRwo8PMs7nmk= -k8s.io/api v0.0.0-20230718170521-0c7a73e2a952/go.mod h1:DlNtgYu2BX/uM/IkiPaFRqDBVazxK5ggJs19ERbMLZ4= -k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966 h1:IYMQQBHbPi6jyorsuYWXB3+kJFH/MxVyLvKJH4nITrg= -k8s.io/apimachinery v0.0.0-20230718054246-5cb236977966/go.mod h1:xhQIsaL3hXneGluH+0pzF7kr+VYuLS/VcYJxF1xQf+g= -k8s.io/client-go v0.0.0-20230718174459-8d42d155509f h1:3Fc11xFTriyNvnOF3LclaZ5hZhwlN/JT8UnFVp5F6wg= -k8s.io/client-go v0.0.0-20230718174459-8d42d155509f/go.mod h1:EKDvjdvs2uH7b2fG8MAqJs37dt7E9oAJJol939kgL5k= -k8s.io/component-base v0.0.0-20230718060631-a69210f80f53 h1:YwgpScqW4EwkVxselbAyOjsNm5lxqJRBhzp0acQhGyo= -k8s.io/component-base v0.0.0-20230718060631-a69210f80f53/go.mod h1:CWOpOwNtaa1/FdU2/DYyPZgGW/VWTWzNb6uXSwoCA5s= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.0.0-20230718060921-6359f9b018b9 h1:PklkOaNcMO6G8YMj82VR4+fJuu2Ds4RJ1fKqFsn/cvM= -k8s.io/kms v0.0.0-20230718060921-6359f9b018b9/go.mod h1:pa8cdjvp+towzePmbH/bxaqmY2rTNWr3XhOuCug8hy4= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= diff --git a/pkg/reconcilers/peer_endpoint_lease.go b/pkg/reconcilers/peer_endpoint_lease.go new file mode 100644 index 000000000..4da16d4f1 --- /dev/null +++ b/pkg/reconcilers/peer_endpoint_lease.go @@ -0,0 +1,364 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconcilers + +import ( + "context" + "fmt" + "net" + "net/http" + "path" + "strconv" + "sync" + "sync/atomic" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/storagebackend" + storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory" +) + +const ( + APIServerIdentityLabel = "apiserverIdentity" +) + +type PeerAdvertiseAddress struct { + PeerAdvertiseIP string + PeerAdvertisePort string +} + +type peerEndpointLeases struct { + storage storage.Interface + destroyFn func() + baseKey string + leaseTime time.Duration +} + +type PeerEndpointLeaseReconciler interface { + // GetEndpoint retrieves the endpoint for a given apiserverId + GetEndpoint(serverId string) (string, error) + // UpdateLease updates the ip and port of peer servers + UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error + // RemoveEndpoints removes this apiserver's peer endpoint lease. + RemoveLease(serverId string) error + // Destroy cleans up everything on shutdown. + Destroy() + // StopReconciling turns any later ReconcileEndpoints call into a noop. + StopReconciling() +} + +type peerEndpointLeaseReconciler struct { + serverLeases *peerEndpointLeases + stopReconcilingCalled atomic.Bool +} + +// NewPeerEndpointLeaseReconciler creates a new peer endpoint lease reconciler +func NewPeerEndpointLeaseReconciler(config *storagebackend.ConfigForResource, baseKey string, leaseTime time.Duration) (PeerEndpointLeaseReconciler, error) { + leaseStorage, destroyFn, err := storagefactory.Create(*config, nil) + if err != nil { + return nil, fmt.Errorf("error creating storage factory: %v", err) + } + var once sync.Once + return &peerEndpointLeaseReconciler{ + serverLeases: &peerEndpointLeases{ + storage: leaseStorage, + destroyFn: func() { once.Do(destroyFn) }, + baseKey: baseKey, + leaseTime: leaseTime, + }, + }, nil +} + +// PeerEndpointController is the controller manager for updating the peer endpoint leases. +// This provides a separate independent reconciliation loop for peer endpoint leases +// which ensures that the peer kube-apiservers are fetching the updated endpoint info for a given apiserver +// in the case when the peer wants to proxy the request to the given apiserver because it can not serve the +// request itself due to version mismatch. +type PeerEndpointLeaseController struct { + reconciler PeerEndpointLeaseReconciler + endpointInterval time.Duration + serverId string + // peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to + // route request to this apiserver in case of a version skew. + peeraddress string + + client kubernetes.Interface + + lock sync.Mutex + stopCh chan struct{} // closed by Stop() +} + +func New(serverId string, peeraddress string, + reconciler PeerEndpointLeaseReconciler, endpointInterval time.Duration, client kubernetes.Interface) *PeerEndpointLeaseController { + return &PeerEndpointLeaseController{ + reconciler: reconciler, + serverId: serverId, + // peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to + // route request to this apiserver in case of a version skew. + peeraddress: peeraddress, + endpointInterval: endpointInterval, + client: client, + stopCh: make(chan struct{}), + } +} + +// Start begins the peer endpoint lease reconciler loop that must exist for bootstrapping +// a cluster. +func (c *PeerEndpointLeaseController) Start(stopCh <-chan struct{}) { + localStopCh := make(chan struct{}) + go func() { + defer close(localStopCh) + select { + case <-stopCh: // from Start + case <-c.stopCh: // from Stop + } + }() + go c.Run(localStopCh) +} + +// RunPeerEndpointReconciler periodically updates the peer endpoint leases +func (c *PeerEndpointLeaseController) Run(stopCh <-chan struct{}) { + // wait until process is ready + wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) { + var code int + c.client.CoreV1().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&code) + return code == http.StatusOK, nil + }, stopCh) + + wait.NonSlidingUntil(func() { + if err := c.UpdatePeerEndpointLeases(); err != nil { + runtime.HandleError(fmt.Errorf("unable to update peer endpoint leases: %v", err)) + } + }, c.endpointInterval, stopCh) +} + +// Stop cleans up this apiserver's peer endpoint leases. +func (c *PeerEndpointLeaseController) Stop() { + c.lock.Lock() + defer c.lock.Unlock() + + select { + case <-c.stopCh: + return // only close once + default: + close(c.stopCh) + } + finishedReconciling := make(chan struct{}) + go func() { + defer close(finishedReconciling) + klog.Infof("Shutting down peer endpoint lease reconciler") + // stop reconciliation + c.reconciler.StopReconciling() + + // Ensure that there will be no race condition with the ReconcileEndpointLeases. + if err := c.reconciler.RemoveLease(c.serverId); err != nil { + klog.Errorf("Unable to remove peer endpoint leases: %v", err) + } + c.reconciler.Destroy() + }() + + select { + case <-finishedReconciling: + // done + case <-time.After(2 * c.endpointInterval): + // don't block server shutdown forever if we can't reach etcd to remove ourselves + klog.Warning("peer_endpoint_controller's RemoveEndpoints() timed out") + } +} + +// UpdatePeerEndpointLeases attempts to update the peer endpoint leases. +func (c *PeerEndpointLeaseController) UpdatePeerEndpointLeases() error { + host, port, err := net.SplitHostPort(c.peeraddress) + if err != nil { + return err + } + + p, err := strconv.Atoi(port) + if err != nil { + return err + } + endpointPorts := createEndpointPortSpec(p, "https") + + // Ensure that there will be no race condition with the RemoveEndpointLeases. + c.lock.Lock() + defer c.lock.Unlock() + + // Refresh the TTL on our key, independently of whether any error or + // update conflict happens below. This makes sure that at least some of + // the servers will add our endpoint lease. + if err := c.reconciler.UpdateLease(c.serverId, host, endpointPorts); err != nil { + return err + } + return nil +} + +// UpdateLease resets the TTL on a server IP in storage +// UpdateLease will create a new key if it doesn't exist. +// We use the first element in endpointPorts as a part of the lease's base key +// This is done to support out tests that simulate 2 apiservers running on the same ip but +// different ports + +// It will also do the following if UnknownVersionInteroperabilityProxy feature is enabled +// 1. store the apiserverId as a label +// 2. store the values passed to --peer-advertise-ip and --peer-advertise-port flags to kube-apiserver as an annotation +// with value of format +func (r *peerEndpointLeaseReconciler) UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error { + // reconcile endpoints only if apiserver was not shutdown + if r.stopReconcilingCalled.Load() { + return nil + } + + // we use the serverID as the key to avoid using the server IP, port as the key. + // note: this means that this lease doesn't enforce mutual exclusion of ip/port usage between apiserver. + key := path.Join(r.serverLeases.baseKey, serverId) + return r.serverLeases.storage.GuaranteedUpdate(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, true, nil, func(input kruntime.Object, respMeta storage.ResponseMeta) (kruntime.Object, *uint64, error) { + existing := input.(*corev1.Endpoints) + existing.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{{IP: ip}}, + Ports: endpointPorts, + }, + } + + // store this server's identity (serverId) as a label. This will be used by + // peers to find the IP of this server when the peer can not serve a request + // due to version skew. + if existing.Labels == nil { + existing.Labels = map[string]string{} + } + existing.Labels[APIServerIdentityLabel] = serverId + + // leaseTime needs to be in seconds + leaseTime := uint64(r.serverLeases.leaseTime / time.Second) + + // NB: GuaranteedUpdate does not perform the store operation unless + // something changed between load and store (not including resource + // version), meaning we can't refresh the TTL without actually + // changing a field. + existing.Generation++ + + klog.V(6).Infof("Resetting TTL on server IP %q listed in storage to %v", ip, leaseTime) + return existing, &leaseTime, nil + }, nil) +} + +// ListLeases retrieves a list of the current server IPs from storage +func (r *peerEndpointLeaseReconciler) ListLeases() ([]string, error) { + storageOpts := storage.ListOptions{ + ResourceVersion: "0", + ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, + Predicate: storage.Everything, + Recursive: true, + } + ipInfoList, err := r.getIpInfoList(storageOpts) + if err != nil { + return nil, err + } + ipList := make([]string, 0, len(ipInfoList.Items)) + for _, ip := range ipInfoList.Items { + if len(ip.Subsets) > 0 && len(ip.Subsets[0].Addresses) > 0 && len(ip.Subsets[0].Addresses[0].IP) > 0 { + ipList = append(ipList, ip.Subsets[0].Addresses[0].IP) + } + } + klog.V(6).Infof("Current server IPs listed in storage are %v", ipList) + return ipList, nil +} + +// GetLease retrieves the server IP and port for a specific server id +func (r *peerEndpointLeaseReconciler) GetLease(serverId string) (string, error) { + var fullAddr string + if serverId == "" { + return "", fmt.Errorf("error getting endpoint for serverId: empty serverId") + } + storageOpts := storage.ListOptions{ + ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, + Predicate: storage.Everything, + Recursive: true, + } + ipInfoList, err := r.getIpInfoList(storageOpts) + if err != nil { + return "", err + } + + for _, ip := range ipInfoList.Items { + if ip.Labels[APIServerIdentityLabel] == serverId { + if len(ip.Subsets) > 0 { + var ipStr, portStr string + if len(ip.Subsets[0].Addresses) > 0 { + if len(ip.Subsets[0].Addresses[0].IP) > 0 { + ipStr = ip.Subsets[0].Addresses[0].IP + } + } + if len(ip.Subsets[0].Ports) > 0 { + portStr = fmt.Sprint(ip.Subsets[0].Ports[0].Port) + } + fullAddr = net.JoinHostPort(ipStr, portStr) + break + } + } + } + klog.V(6).Infof("Fetched this server IP for the specified apiserverId %v, %v", serverId, fullAddr) + return fullAddr, nil +} + +func (r *peerEndpointLeaseReconciler) StopReconciling() { + r.stopReconcilingCalled.Store(true) +} + +// RemoveLease removes the lease on a server IP in storage +// We use the first element in endpointPorts as a part of the lease's base key +// This is done to support out tests that simulate 2 apiservers running on the same ip but +// different ports +func (r *peerEndpointLeaseReconciler) RemoveLease(serverId string) error { + key := path.Join(r.serverLeases.baseKey, serverId) + return r.serverLeases.storage.Delete(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, nil, rest.ValidateAllObjectFunc, nil) +} + +func (r *peerEndpointLeaseReconciler) Destroy() { + r.serverLeases.destroyFn() +} + +func (r *peerEndpointLeaseReconciler) GetEndpoint(serverId string) (string, error) { + return r.GetLease(serverId) +} + +func (r *peerEndpointLeaseReconciler) getIpInfoList(storageOpts storage.ListOptions) (*corev1.EndpointsList, error) { + ipInfoList := &corev1.EndpointsList{} + if err := r.serverLeases.storage.GetList(apirequest.NewDefaultContext(), r.serverLeases.baseKey, storageOpts, ipInfoList); err != nil { + return nil, err + } + return ipInfoList, nil +} + +// createEndpointPortSpec creates the endpoint ports +func createEndpointPortSpec(endpointPort int, endpointPortName string) []corev1.EndpointPort { + return []corev1.EndpointPort{{ + Protocol: corev1.ProtocolTCP, + Port: int32(endpointPort), + Name: endpointPortName, + }} +} diff --git a/pkg/reconcilers/peer_endpoint_lease_test.go b/pkg/reconcilers/peer_endpoint_lease_test.go new file mode 100644 index 000000000..07e5f1e9c --- /dev/null +++ b/pkg/reconcilers/peer_endpoint_lease_test.go @@ -0,0 +1,278 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconcilers + +import ( + "reflect" + "sort" + "testing" + "time" + + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/apitesting" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/storage" + etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" + "k8s.io/apiserver/pkg/storage/storagebackend/factory" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" +) + +func init() { + var scheme = runtime.NewScheme() + + metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion)) + + codecs = serializer.NewCodecFactory(scheme) +} + +var codecs serializer.CodecFactory + +type serverInfo struct { + existingIP string + id string + ports []corev1.EndpointPort + newIP string + removeLease bool + expectEndpoint string +} + +func NewFakePeerEndpointReconciler(t *testing.T, s storage.Interface) peerEndpointLeaseReconciler { + // use the same base key used by the controlplane, but add a random + // prefix so we can reuse the etcd instance for subtests independently. + base := "/" + uuid.New().String() + "/peerserverleases/" + return peerEndpointLeaseReconciler{serverLeases: &peerEndpointLeases{ + storage: s, + destroyFn: func() {}, + baseKey: base, + leaseTime: 1 * time.Minute, // avoid the lease to timeout on tests + }} +} + +func (f *peerEndpointLeaseReconciler) SetKeys(servers []serverInfo) error { + for _, server := range servers { + if err := f.UpdateLease(server.id, server.existingIP, server.ports); err != nil { + return err + } + } + return nil +} + +func TestPeerEndpointLeaseReconciler(t *testing.T) { + // enable feature flags + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)() + + server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) + t.Cleanup(func() { server.Terminate(t) }) + + newFunc := func() runtime.Object { return &corev1.Endpoints{} } + sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion) + + s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "endpoints"}), newFunc) + if err != nil { + t.Fatalf("Error creating storage: %v", err) + } + t.Cleanup(dFunc) + + tests := []struct { + testName string + servers []serverInfo + expectLeases []string + }{ + { + testName: "existing IP satisfy", + servers: []serverInfo{{ + existingIP: "4.3.2.1", + id: "server-1", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + expectEndpoint: "4.3.2.1:8080", + }, { + existingIP: "1.2.3.4", + id: "server-2", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + expectEndpoint: "1.2.3.4:8080", + }}, + expectLeases: []string{"4.3.2.1", "1.2.3.4"}, + }, + { + testName: "existing IP + new IP = should return the new IP", + servers: []serverInfo{{ + existingIP: "4.3.2.2", + id: "server-1", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + newIP: "4.3.2.1", + expectEndpoint: "4.3.2.1:8080", + }, { + existingIP: "1.2.3.4", + id: "server-2", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + newIP: "1.1.1.1", + expectEndpoint: "1.1.1.1:8080", + }}, + expectLeases: []string{"4.3.2.1", "1.1.1.1"}, + }, + { + testName: "no existing IP, should return new IP", + servers: []serverInfo{{ + id: "server-1", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + newIP: "1.2.3.4", + expectEndpoint: "1.2.3.4:8080", + }}, + expectLeases: []string{"1.2.3.4"}, + }, + } + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + fakeReconciler := NewFakePeerEndpointReconciler(t, s) + err := fakeReconciler.SetKeys(test.servers) + if err != nil { + t.Errorf("unexpected error creating keys: %v", err) + } + + for _, server := range test.servers { + if server.newIP != "" { + err = fakeReconciler.UpdateLease(server.id, server.newIP, server.ports) + if err != nil { + t.Errorf("unexpected error reconciling: %v", err) + } + } + } + + leases, err := fakeReconciler.ListLeases() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + // sort for comparison + sort.Strings(leases) + sort.Strings(test.expectLeases) + if !reflect.DeepEqual(leases, test.expectLeases) { + t.Errorf("expected %v got: %v", test.expectLeases, leases) + } + + for _, server := range test.servers { + endpoint, err := fakeReconciler.GetLease(server.id) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if endpoint != server.expectEndpoint { + t.Errorf("expected %v got: %v", server.expectEndpoint, endpoint) + } + } + }) + } +} + +func TestPeerLeaseRemoveEndpoints(t *testing.T) { + // enable feature flags + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)() + + server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) + t.Cleanup(func() { server.Terminate(t) }) + + newFunc := func() runtime.Object { return &corev1.Endpoints{} } + sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion) + + s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), newFunc) + if err != nil { + t.Fatalf("Error creating storage: %v", err) + } + t.Cleanup(dFunc) + + stopTests := []struct { + testName string + servers []serverInfo + expectLeases []string + apiServerStartup bool + }{ + { + testName: "successful remove previous endpoints before apiserver starts", + servers: []serverInfo{ + { + existingIP: "1.2.3.4", + id: "test-server-1", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + removeLease: true, + }, + { + existingIP: "2.4.6.8", + id: "test-server-2", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + }}, + expectLeases: []string{"2.4.6.8"}, + apiServerStartup: true, + }, + { + testName: "stop reconciling with new IP not in existing ip list", + servers: []serverInfo{{ + existingIP: "1.2.3.4", + newIP: "4.6.8.9", + id: "test-server-1", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + }, + { + existingIP: "2.4.6.8", + id: "test-server-2", + ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + removeLease: true, + }}, + expectLeases: []string{"1.2.3.4"}, + }, + } + for _, test := range stopTests { + t.Run(test.testName, func(t *testing.T) { + fakeReconciler := NewFakePeerEndpointReconciler(t, s) + err := fakeReconciler.SetKeys(test.servers) + if err != nil { + t.Errorf("unexpected error creating keys: %v", err) + } + if !test.apiServerStartup { + fakeReconciler.StopReconciling() + } + for _, server := range test.servers { + if server.removeLease { + err = fakeReconciler.RemoveLease(server.id) + // if the ip is not on the endpoints, it must return an storage error and stop reconciling + if err != nil { + t.Errorf("unexpected error reconciling: %v", err) + } + } + } + + leases, err := fakeReconciler.ListLeases() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + // sort for comparison + sort.Strings(leases) + sort.Strings(test.expectLeases) + if !reflect.DeepEqual(leases, test.expectLeases) { + t.Errorf("expected %v got: %v", test.expectLeases, leases) + } + }) + } +} diff --git a/pkg/server/config.go b/pkg/server/config.go index c9d449572..d678f52df 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -86,6 +86,13 @@ import ( _ "k8s.io/apiserver/pkg/apis/apiserver/install" ) +// hostnameFunc is a function to set the hostnameFunc of this apiserver. +// To be used for testing purpose only, to simulate scenarios where multiple apiservers +// exist. In such cases we want to ensure unique apiserver IDs which are a hash of hostnameFunc. +var ( + hostnameFunc = os.Hostname +) + const ( // DefaultLegacyAPIPrefix is where the legacy APIs will be located. DefaultLegacyAPIPrefix = "/api" @@ -367,7 +374,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config { defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz} var id string if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { - hostname, err := os.Hostname() + hostname, err := hostnameFunc() if err != nil { klog.Fatalf("error getting hostname for apiserver identity: %v", err) } @@ -897,7 +904,9 @@ func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c } func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { - handler := filterlatency.TrackCompleted(apiHandler) + handler := apiHandler + + handler = filterlatency.TrackCompleted(handler) handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer) handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization") @@ -1070,3 +1079,12 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences) authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator) } + +// For testing purpose only +func SetHostnameFuncForTests(name string) { + hostnameFunc = func() (host string, err error) { + host = name + err = nil + return + } +} diff --git a/pkg/server/routes/metrics.go b/pkg/server/routes/metrics.go index d30f74b9c..ad1eb2835 100644 --- a/pkg/server/routes/metrics.go +++ b/pkg/server/routes/metrics.go @@ -22,6 +22,7 @@ import ( cachermetrics "k8s.io/apiserver/pkg/storage/cacher/metrics" etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics" flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics" + peerproxymetrics "k8s.io/apiserver/pkg/util/peerproxy/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -50,4 +51,5 @@ func register() { cachermetrics.Register() etcd3metrics.Register() flowcontrolmetrics.Register() + peerproxymetrics.Register() } diff --git a/pkg/util/peerproxy/metrics/metrics.go b/pkg/util/peerproxy/metrics/metrics.go new file mode 100644 index 000000000..48b89be75 --- /dev/null +++ b/pkg/util/peerproxy/metrics/metrics.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "sync" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const ( + subsystem = "apiserver" + statuscode = "code" +) + +var registerMetricsOnce sync.Once + +var ( + // peerProxiedRequestsTotal counts the number of requests that were proxied to a peer kube-apiserver. + peerProxiedRequestsTotal = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: subsystem, + Name: "rerouted_request_total", + Help: "Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it", + StabilityLevel: metrics.ALPHA, + }, + []string{statuscode}, + ) +) + +func Register() { + registerMetricsOnce.Do(func() { + legacyregistry.MustRegister(peerProxiedRequestsTotal) + }) +} + +// IncPeerProxiedRequest increments the # of proxied requests to peer kube-apiserver +func IncPeerProxiedRequest(ctx context.Context, status string) { + peerProxiedRequestsTotal.WithContext(ctx).WithLabelValues(status).Add(1) +} diff --git a/pkg/util/peerproxy/peerproxy.go b/pkg/util/peerproxy/peerproxy.go new file mode 100644 index 000000000..7ea4d5c4b --- /dev/null +++ b/pkg/util/peerproxy/peerproxy.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package peerproxy + +import ( + "net/http" + "sync" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/reconcilers" + "k8s.io/apiserver/pkg/storageversion" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" +) + +// Interface defines how the Unknown Version Proxy filter interacts with the underlying system. +type Interface interface { + WrapHandler(handler http.Handler) http.Handler + WaitForCacheSync(stopCh <-chan struct{}) error + HasFinishedSync() bool +} + +// New creates a new instance to implement unknown version proxy +func NewPeerProxyHandler(informerFactory kubeinformers.SharedInformerFactory, + svm storageversion.Manager, + proxyTransport http.RoundTripper, + serverId string, + reconciler reconcilers.PeerEndpointLeaseReconciler, + serializer runtime.NegotiatedSerializer) *peerProxyHandler { + h := &peerProxyHandler{ + name: "PeerProxyHandler", + storageversionManager: svm, + proxyTransport: proxyTransport, + svMap: sync.Map{}, + serverId: serverId, + reconciler: reconciler, + serializer: serializer, + } + svi := informerFactory.Internal().V1alpha1().StorageVersions() + h.storageversionInformer = svi.Informer() + + svi.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + h.addSV(obj) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + h.updateSV(oldObj, newObj) + }, + DeleteFunc: func(obj interface{}) { + h.deleteSV(obj) + }}) + return h +} diff --git a/pkg/util/peerproxy/peerproxy_handler.go b/pkg/util/peerproxy/peerproxy_handler.go new file mode 100644 index 000000000..bc342165b --- /dev/null +++ b/pkg/util/peerproxy/peerproxy_handler.go @@ -0,0 +1,357 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package peerproxy + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + + "k8s.io/api/apiserverinternal/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/proxy" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" + epmetrics "k8s.io/apiserver/pkg/endpoints/metrics" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/endpoints/responsewriter" + "k8s.io/apiserver/pkg/reconcilers" + "k8s.io/apiserver/pkg/storageversion" + "k8s.io/apiserver/pkg/util/peerproxy/metrics" + apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/transport" + "k8s.io/klog/v2" +) + +const ( + PeerProxiedHeader = "x-kubernetes-peer-proxied" +) + +type peerProxyHandler struct { + name string + // StorageVersion informer used to fetch apiserver ids than can serve a resource + storageversionInformer cache.SharedIndexInformer + + // StorageVersion manager used to ensure it has finished updating storageversions before + // we start handling external requests + storageversionManager storageversion.Manager + + // proxy transport + proxyTransport http.RoundTripper + + // identity for this server + serverId string + + // reconciler that is used to fetch host port of peer apiserver when proxying request to a peer + reconciler reconcilers.PeerEndpointLeaseReconciler + + serializer runtime.NegotiatedSerializer + + // SyncMap for storing an up to date copy of the storageversions and apiservers that can serve them + // This map is populated using the StorageVersion informer + // This map has key set to GVR and value being another SyncMap + // The nested SyncMap has key set to apiserver id and value set to boolean + // The nested maps are created to have a "Set" like structure to store unique apiserver ids + // for a given GVR + svMap sync.Map + + finishedSync atomic.Bool +} + +type serviceableByResponse struct { + locallyServiceable bool + errorFetchingAddressFromLease bool + peerEndpoints []string +} + +// responder implements rest.Responder for assisting a connector in writing objects or errors. +type responder struct { + w http.ResponseWriter + ctx context.Context +} + +func (h *peerProxyHandler) HasFinishedSync() bool { + return h.finishedSync.Load() +} + +func (h *peerProxyHandler) WaitForCacheSync(stopCh <-chan struct{}) error { + + ok := cache.WaitForNamedCacheSync("unknown-version-proxy", stopCh, h.storageversionInformer.HasSynced, h.storageversionManager.Completed) + if !ok { + return fmt.Errorf("error while waiting for initial cache sync") + } + klog.V(3).Infof("setting finishedSync to true") + h.finishedSync.Store(true) + return nil +} + +// WrapHandler will fetch the apiservers that can serve the request and either serve it locally +// or route it to a peer +func (h *peerProxyHandler) WrapHandler(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + requestInfo, ok := apirequest.RequestInfoFrom(ctx) + + if !ok { + responsewriters.InternalError(w, r, errors.New("no RequestInfo found in the context")) + return + } + + // Allow non-resource requests + if !requestInfo.IsResourceRequest { + klog.V(3).Infof("Not a resource request skipping proxying") + handler.ServeHTTP(w, r) + return + } + + // Request has already been proxied once, it must be served locally + if r.Header.Get(PeerProxiedHeader) == "true" { + klog.V(3).Infof("Already rerouted once, skipping proxying to peer") + handler.ServeHTTP(w, r) + return + } + + // StorageVersion Informers and/or StorageVersionManager is not synced yet, pass request to next handler + // This will happen for self requests from the kube-apiserver because we have a poststarthook + // to ensure that external requests are not served until the StorageVersion Informer and + // StorageVersionManager has synced + if !h.HasFinishedSync() { + handler.ServeHTTP(w, r) + return + } + + gvr := schema.GroupVersionResource{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion, Resource: requestInfo.Resource} + if requestInfo.APIGroup == "" { + gvr.Group = "core" + } + + // find servers that are capable of serving this request + serviceableByResp, err := h.findServiceableByServers(gvr, h.serverId, h.reconciler) + if err != nil { + // this means that resource is an aggregated API or a CR since it wasn't found in SV informer cache, pass as it is + handler.ServeHTTP(w, r) + return + } + // found the gvr locally, pass request to the next handler in local apiserver + if serviceableByResp.locallyServiceable { + handler.ServeHTTP(w, r) + return + } + + gv := schema.GroupVersion{Group: gvr.Group, Version: gvr.Version} + + if serviceableByResp.errorFetchingAddressFromLease { + klog.ErrorS(err, "error fetching ip and port of remote server while proxying") + responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable("Error getting ip and port info of the remote server while proxying"), h.serializer, gv, w, r) + return + } + + // no apiservers were found that could serve the request, pass request to + // next handler, that should eventually serve 404 + + // TODO: maintain locally serviceable GVRs somewhere so that we dont have to + // consult the storageversion-informed map for those + if len(serviceableByResp.peerEndpoints) == 0 { + klog.Errorf(fmt.Sprintf("GVR %v is not served by anything in this cluster", gvr)) + handler.ServeHTTP(w, r) + return + } + + // otherwise, randomly select an apiserver and proxy request to it + rand := rand.Intn(len(serviceableByResp.peerEndpoints)) + destServerHostPort := serviceableByResp.peerEndpoints[rand] + h.proxyRequestToDestinationAPIServer(r, w, destServerHostPort) + + }) +} + +func (h *peerProxyHandler) findServiceableByServers(gvr schema.GroupVersionResource, localAPIServerId string, reconciler reconcilers.PeerEndpointLeaseReconciler) (serviceableByResponse, error) { + + apiserversi, ok := h.svMap.Load(gvr) + + // no value found for the requested gvr in svMap + if !ok || apiserversi == nil { + return serviceableByResponse{}, fmt.Errorf("no StorageVersions found for the GVR: %v", gvr) + } + apiservers := apiserversi.(*sync.Map) + response := serviceableByResponse{} + var peerServerEndpoints []string + apiservers.Range(func(key, value interface{}) bool { + apiserverKey := key.(string) + if apiserverKey == localAPIServerId { + response.errorFetchingAddressFromLease = true + response.locallyServiceable = true + // stop iteration + return false + } + + hostPort, err := reconciler.GetEndpoint(apiserverKey) + if err != nil { + response.errorFetchingAddressFromLease = true + klog.Errorf("failed to get peer ip from storage lease for server %s", apiserverKey) + // continue with iteration + return true + } + // check ip format + _, _, err = net.SplitHostPort(hostPort) + if err != nil { + response.errorFetchingAddressFromLease = true + klog.Errorf("invalid address found for server %s", apiserverKey) + // continue with iteration + return true + } + peerServerEndpoints = append(peerServerEndpoints, hostPort) + // continue with iteration + return true + }) + + response.peerEndpoints = peerServerEndpoints + return response, nil +} + +func (h *peerProxyHandler) proxyRequestToDestinationAPIServer(req *http.Request, rw http.ResponseWriter, host string) { + user, ok := apirequest.UserFrom(req.Context()) + if !ok { + klog.Errorf("failed to get user info from request") + return + } + + // write a new location based on the existing request pointed at the target service + location := &url.URL{} + location.Scheme = "https" + location.Host = host + location.Path = req.URL.Path + location.RawQuery = req.URL.Query().Encode() + + newReq, cancelFn := apiserverproxyutil.NewRequestForProxy(location, req) + newReq.Header.Add(PeerProxiedHeader, "true") + defer cancelFn() + + proxyRoundTripper := transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), h.proxyTransport) + + delegate := &epmetrics.ResponseWriterDelegator{ResponseWriter: rw} + w := responsewriter.WrapForHTTP1Or2(delegate) + + handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, false, &responder{w: w, ctx: req.Context()}) + handler.ServeHTTP(w, newReq) + // Increment the count of proxied requests + metrics.IncPeerProxiedRequest(req.Context(), strconv.Itoa(delegate.Status())) +} + +func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) { + klog.Errorf("Error while proxying request to destination apiserver: %v", err) + http.Error(w, err.Error(), http.StatusServiceUnavailable) +} + +// Adds a storageversion object to SVMap +func (h *peerProxyHandler) addSV(obj interface{}) { + sv, ok := obj.(*v1alpha1.StorageVersion) + if !ok { + klog.Errorf("Invalid StorageVersion provided to addSV()") + return + } + h.updateSVMap(nil, sv) +} + +// Updates the SVMap to delete old storageversion and add new storageversion +func (h *peerProxyHandler) updateSV(oldObj interface{}, newObj interface{}) { + oldSV, ok := oldObj.(*v1alpha1.StorageVersion) + if !ok { + klog.Errorf("Invalid StorageVersion provided to updateSV()") + return + } + newSV, ok := newObj.(*v1alpha1.StorageVersion) + if !ok { + klog.Errorf("Invalid StorageVersion provided to updateSV()") + return + } + h.updateSVMap(oldSV, newSV) +} + +// Deletes a storageversion object from SVMap +func (h *peerProxyHandler) deleteSV(obj interface{}) { + sv, ok := obj.(*v1alpha1.StorageVersion) + if !ok { + klog.Errorf("Invalid StorageVersion provided to deleteSV()") + return + } + h.updateSVMap(sv, nil) +} + +// Delete old storageversion, add new storagversion +func (h *peerProxyHandler) updateSVMap(oldSV *v1alpha1.StorageVersion, newSV *v1alpha1.StorageVersion) { + if oldSV != nil { + // delete old SV entries + h.deleteSVFromMap(oldSV) + } + if newSV != nil { + // add new SV entries + h.addSVToMap(newSV) + } +} + +func (h *peerProxyHandler) deleteSVFromMap(sv *v1alpha1.StorageVersion) { + // The name of storageversion is . + splitInd := strings.LastIndex(sv.Name, ".") + group := sv.Name[:splitInd] + resource := sv.Name[splitInd+1:] + + gvr := schema.GroupVersionResource{Group: group, Resource: resource} + for _, gr := range sv.Status.StorageVersions { + for _, version := range gr.ServedVersions { + versionSplit := strings.Split(version, "/") + if len(versionSplit) == 2 { + version = versionSplit[1] + } + gvr.Version = version + h.svMap.Delete(gvr) + } + } +} + +func (h *peerProxyHandler) addSVToMap(sv *v1alpha1.StorageVersion) { + // The name of storageversion is . + splitInd := strings.LastIndex(sv.Name, ".") + group := sv.Name[:splitInd] + resource := sv.Name[splitInd+1:] + + gvr := schema.GroupVersionResource{Group: group, Resource: resource} + for _, gr := range sv.Status.StorageVersions { + for _, version := range gr.ServedVersions { + + // some versions have groups included in them, so get rid of the groups + versionSplit := strings.Split(version, "/") + if len(versionSplit) == 2 { + version = versionSplit[1] + } + gvr.Version = version + apiserversi, _ := h.svMap.LoadOrStore(gvr, &sync.Map{}) + apiservers := apiserversi.(*sync.Map) + apiservers.Store(gr.APIServerID, true) + } + } +} diff --git a/pkg/util/peerproxy/peerproxy_handler_test.go b/pkg/util/peerproxy/peerproxy_handler_test.go new file mode 100644 index 000000000..9b1cc9b5e --- /dev/null +++ b/pkg/util/peerproxy/peerproxy_handler_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package peerproxy + +import ( + "net/http" + "strings" + "sync" + "testing" + "time" + + "net/http/httptest" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/apitesting" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authentication/user" + apifilters "k8s.io/apiserver/pkg/endpoints/filters" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/reconcilers" + etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" + "k8s.io/apiserver/pkg/storageversion" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/peerproxy/metrics" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/transport" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" +) + +const ( + requestTimeout = 30 * time.Second + localServerId = "local-apiserver" + remoteServerId = "remote-apiserver" +) + +type FakeSVMapData struct { + gvr schema.GroupVersionResource + serverId string +} + +type reconciler struct { + do bool + publicIP string + serverId string +} + +func TestPeerProxy(t *testing.T) { + testCases := []struct { + desc string + svdata FakeSVMapData + informerFinishedSync bool + requestPath string + peerproxiedHeader string + expectedStatus int + metrics []string + want string + reconcilerConfig reconciler + }{ + { + desc: "allow non resource requests", + requestPath: "/foo/bar/baz", + expectedStatus: http.StatusOK, + }, + { + desc: "allow if already proxied once", + requestPath: "/api/bar/baz", + expectedStatus: http.StatusOK, + peerproxiedHeader: "true", + }, + { + desc: "allow if unsynced informers", + requestPath: "/api/bar/baz", + expectedStatus: http.StatusOK, + informerFinishedSync: false, + }, + { + desc: "allow if no storage version found", + requestPath: "/api/bar/baz", + expectedStatus: http.StatusOK, + informerFinishedSync: true, + }, + { + // since if no server id is found, we pass request to next handler + //, and our last handler in local chain is an http ok handler + desc: "200 if no serverid found", + requestPath: "/api/bar/baz", + expectedStatus: http.StatusOK, + informerFinishedSync: true, + svdata: FakeSVMapData{ + gvr: schema.GroupVersionResource{ + Group: "core", + Version: "bar", + Resource: "baz"}, + serverId: ""}, + }, + { + desc: "503 if no endpoint fetched from lease", + requestPath: "/api/foo/bar", + expectedStatus: http.StatusServiceUnavailable, + informerFinishedSync: true, + svdata: FakeSVMapData{ + gvr: schema.GroupVersionResource{ + Group: "core", + Version: "foo", + Resource: "bar"}, + serverId: remoteServerId}, + }, + { + desc: "200 if locally serviceable", + requestPath: "/api/foo/bar", + expectedStatus: http.StatusOK, + informerFinishedSync: true, + svdata: FakeSVMapData{ + gvr: schema.GroupVersionResource{ + Group: "core", + Version: "foo", + Resource: "bar"}, + serverId: localServerId}, + }, + { + desc: "503 unreachable peer bind address", + requestPath: "/api/foo/bar", + expectedStatus: http.StatusServiceUnavailable, + informerFinishedSync: true, + svdata: FakeSVMapData{ + gvr: schema.GroupVersionResource{ + Group: "core", + Version: "foo", + Resource: "bar"}, + serverId: remoteServerId}, + reconcilerConfig: reconciler{ + do: true, + publicIP: "1.2.3.4", + serverId: remoteServerId, + }, + metrics: []string{ + "apiserver_rerouted_request_total", + }, + want: ` + # HELP apiserver_rerouted_request_total [ALPHA] Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it + # TYPE apiserver_rerouted_request_total counter + apiserver_rerouted_request_total{code="503"} 1 + `, + }, + { + desc: "503 unreachable peer public address", + requestPath: "/api/foo/bar", + expectedStatus: http.StatusServiceUnavailable, + informerFinishedSync: true, + svdata: FakeSVMapData{ + gvr: schema.GroupVersionResource{ + Group: "core", + Version: "foo", + Resource: "bar"}, + serverId: remoteServerId}, + reconcilerConfig: reconciler{ + do: true, + publicIP: "1.2.3.4", + serverId: remoteServerId, + }, + metrics: []string{ + "apiserver_rerouted_request_total", + }, + want: ` + # HELP apiserver_rerouted_request_total [ALPHA] Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it + # TYPE apiserver_rerouted_request_total counter + apiserver_rerouted_request_total{code="503"} 2 + `, + }, + } + + metrics.Register() + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + lastHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("OK")) + }) + reconciler := newFakePeerEndpointReconciler(t) + handler := newHandlerChain(t, lastHandler, reconciler, tt.informerFinishedSync, tt.svdata) + server, requestGetter := createHTTP2ServerWithClient(handler, requestTimeout*2) + defer server.Close() + + if tt.reconcilerConfig.do { + // need to enable feature flags first + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)() + + reconciler.UpdateLease(tt.reconcilerConfig.serverId, + tt.reconcilerConfig.publicIP, + []corev1.EndpointPort{{Name: "foo", + Port: 8080, Protocol: "TCP"}}) + } + + req, err := http.NewRequest(http.MethodGet, server.URL+tt.requestPath, nil) + if err != nil { + t.Fatalf("failed to create new http request - %v", err) + } + req.Header.Set(PeerProxiedHeader, tt.peerproxiedHeader) + + resp, _ := requestGetter(req) + + // compare response + assert.Equal(t, tt.expectedStatus, resp.StatusCode) + + // compare metric + if tt.want != "" { + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { + t.Fatal(err) + } + } + }) + } + +} + +func newFakePeerEndpointReconciler(t *testing.T) reconcilers.PeerEndpointLeaseReconciler { + server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) + t.Cleanup(func() { server.Terminate(t) }) + scheme := runtime.NewScheme() + metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) + //utilruntime.Must(core.AddToScheme(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion)) + codecs := serializer.NewCodecFactory(scheme) + sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion) + config := *sc.ForResource(schema.GroupResource{Resource: "endpoints"}) + baseKey := "/" + uuid.New().String() + "/peer-testleases/" + leaseTime := 1 * time.Minute + reconciler, err := reconcilers.NewPeerEndpointLeaseReconciler(&config, baseKey, leaseTime) + if err != nil { + t.Fatalf("Error creating storage: %v", err) + } + return reconciler +} + +func newHandlerChain(t *testing.T, handler http.Handler, reconciler reconcilers.PeerEndpointLeaseReconciler, informerFinishedSync bool, svdata FakeSVMapData) http.Handler { + // Add peerproxy handler + s := serializer.NewCodecFactory(runtime.NewScheme()).WithoutConversion() + peerProxyHandler, err := newFakePeerProxyHandler(informerFinishedSync, reconciler, svdata, localServerId, s) + if err != nil { + t.Fatalf("Error creating peer proxy handler: %v", err) + } + peerProxyHandler.finishedSync.Store(informerFinishedSync) + handler = peerProxyHandler.WrapHandler(handler) + + // Add user info + handler = withFakeUser(handler) + + // Add requestInfo handler + requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")} + handler = apifilters.WithRequestInfo(handler, requestInfoFactory) + return handler +} + +func newFakePeerProxyHandler(informerFinishedSync bool, reconciler reconcilers.PeerEndpointLeaseReconciler, svdata FakeSVMapData, id string, s runtime.NegotiatedSerializer) (*peerProxyHandler, error) { + clientset := fake.NewSimpleClientset() + informerFactory := informers.NewSharedInformerFactory(clientset, 0) + clientConfig := &transport.Config{ + TLS: transport.TLSConfig{ + Insecure: false, + }} + proxyRoundTripper, err := transport.New(clientConfig) + if err != nil { + return nil, err + } + ppI := NewPeerProxyHandler(informerFactory, storageversion.NewDefaultManager(), proxyRoundTripper, id, reconciler, s) + if testDataExists(svdata.gvr) { + ppI.addToStorageVersionMap(svdata.gvr, svdata.serverId) + } + return ppI, nil +} + +func (h *peerProxyHandler) addToStorageVersionMap(gvr schema.GroupVersionResource, serverId string) { + apiserversi, _ := h.svMap.LoadOrStore(gvr, &sync.Map{}) + apiservers := apiserversi.(*sync.Map) + if serverId != "" { + apiservers.Store(serverId, true) + } +} + +func testDataExists(gvr schema.GroupVersionResource) bool { + return gvr.Group != "" && gvr.Version != "" && gvr.Resource != "" +} + +func withFakeUser(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{ + Groups: r.Header["Groups"], + })) + handler.ServeHTTP(w, r) + }) +} + +// returns a started http2 server, with a client function to send request to the server. +func createHTTP2ServerWithClient(handler http.Handler, clientTimeout time.Duration) (*httptest.Server, func(req *http.Request) (*http.Response, error)) { + server := httptest.NewUnstartedServer(handler) + server.EnableHTTP2 = true + server.StartTLS() + cli := server.Client() + cli.Timeout = clientTimeout + return server, func(req *http.Request) (*http.Response, error) { + return cli.Do(req) + } +} diff --git a/pkg/util/proxy/proxy.go b/pkg/util/proxy/proxy.go index 8f6015cfa..9e0bdfaf0 100644 --- a/pkg/util/proxy/proxy.go +++ b/pkg/util/proxy/proxy.go @@ -17,17 +17,30 @@ limitations under the License. package proxy import ( + "context" "fmt" "math/rand" "net" + "net/http" "net/url" "strconv" + "strings" + "time" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + utilnet "k8s.io/apimachinery/pkg/util/net" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/audit" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" listersv1 "k8s.io/client-go/listers/core/v1" ) +const ( + // taken from https://github.com/kubernetes/kubernetes/blob/release-1.27/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go#L47 + aggregatedDiscoveryTimeout = 5 * time.Second +) + // findServicePort finds the service port by name or numerically. func findServicePort(svc *v1.Service, port int32) (*v1.ServicePort, error) { for _, svcPort := range svc.Spec.Ports { @@ -117,3 +130,34 @@ func ResolveCluster(services listersv1.ServiceLister, namespace, id string, port return nil, fmt.Errorf("unsupported service type %q", svc.Spec.Type) } } + +// NewRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests +func NewRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) { + newCtx := req.Context() + cancelFn := func() {} + + if requestInfo, ok := genericapirequest.RequestInfoFrom(req.Context()); ok { + // trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three + // segments that we are going to proxy, we have a discovery request. + if !requestInfo.IsResourceRequest && len(strings.Split(strings.Trim(requestInfo.Path, "/"), "/")) == 3 { + // discovery requests are used by kubectl and others to determine which resources a server has. This is a cheap call that + // should be fast for every aggregated apiserver. Latency for aggregation is expected to be low (as for all extensions) + // so forcing a short timeout here helps responsiveness of all clients. + newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout) + } + } + + // WithContext creates a shallow clone of the request with the same context. + newReq := req.WithContext(newCtx) + newReq.Header = utilnet.CloneHeader(req.Header) + newReq.URL = location + newReq.Host = location.Host + + // If the original request has an audit ID, let's make sure we propagate this + // to the aggregated server. + if auditID, found := audit.AuditIDFrom(req.Context()); found { + newReq.Header.Set(auditinternal.HeaderAuditID, string(auditID)) + } + + return newReq, cancelFn +}