Merge pull request #113682 from pawbana/add-verb-resource-to-api-server-tracing

Add verb resource to api server tracing

Kubernetes-commit: 887d92d20b3d1538ba900c466ead25ed662179cc

go.mod

@@ -42,7 +42,7 @@ require (
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.2.2
-	k8s.io/api v0.0.0-20221108053744-6c616e191dd8
+	k8s.io/api v0.0.0-20221108053746-72a29bf6f029
 	k8s.io/apimachinery v0.0.0-20221108052756-0ceff9075aa0
 	k8s.io/client-go v0.0.0-20221108054905-7ed3193a72f8
 	k8s.io/component-base v0.0.0-20221104121019-23eee025084e
@@ -122,7 +122,7 @@ require (
 )
 
 replace (
-	k8s.io/api => k8s.io/api v0.0.0-20221108053744-6c616e191dd8
+	k8s.io/api => k8s.io/api v0.0.0-20221108053746-72a29bf6f029
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20221108052756-0ceff9075aa0
 	k8s.io/client-go => k8s.io/client-go v0.0.0-20221108054905-7ed3193a72f8
 	k8s.io/component-base => k8s.io/component-base v0.0.0-20221104121019-23eee025084e

go.sum

@@ -985,8 +985,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.0.0-20221108053744-6c616e191dd8 h1:/CGL/BQFxTEw4P+1gVLV4qHihsIDu0Wuf4df43Gk8cc=
-k8s.io/api v0.0.0-20221108053744-6c616e191dd8/go.mod h1:O7KYltqKbaDavU20bYboF3FKNWyQen8XmDYgb8+Kat0=
+k8s.io/api v0.0.0-20221108053746-72a29bf6f029 h1:pBrvi2/huXT3iDfKvhr3/lVZPZzkUAmYGji7y3HQ72U=
+k8s.io/api v0.0.0-20221108053746-72a29bf6f029/go.mod h1:ZKcZjfFls/EwXe3Ar9LdZy7hp9WLiYbLbhOMuS9WoWw=
 k8s.io/apimachinery v0.0.0-20221108052756-0ceff9075aa0 h1:joX16rSALtkNhZ03fUTByWLcVq1w4JhrOBOGhH3kKRw=
 k8s.io/apimachinery v0.0.0-20221108052756-0ceff9075aa0/go.mod h1:zSkBXgO5G/dSQOe256tx5Yo2OJytojpY3bsXu/4/ZJE=
 k8s.io/client-go v0.0.0-20221108054905-7ed3193a72f8 h1:Bs8yOqX52JMbmgr/Ar3adOXOflYp5XtWJ1f6YOOppq0=

pkg/endpoints/handlers/helpers.go

@@ -21,6 +21,8 @@ import (
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 )
 
 const (
@@ -88,3 +90,34 @@ func (lazy *lazyAuditID) String() string {
 
 	return "unknown"
 }
+
+// lazyVerb implements String() string and it will
+// lazily get normalized Verb
+type lazyVerb struct {
+	req *http.Request
+}
+
+func (lazy *lazyVerb) String() string {
+	if lazy.req == nil {
+		return "unknown"
+	}
+	return metrics.NormalizedVerb(lazy.req)
+}
+
+// lazyResource implements String() string and it will
+// lazily get Resource from request info
+type lazyResource struct {
+	req *http.Request
+}
+
+func (lazy *lazyResource) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.Resource
+		}
+	}
+
+	return "unknown"
+}

pkg/endpoints/handlers/helpers_test.go

@@ -17,10 +17,14 @@ limitations under the License.
 package handlers
 
 import (
+	"context"
 	"fmt"
-	"github.com/stretchr/testify/assert"
 	"net/http"
+	"net/url"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apiserver/pkg/endpoints/request"
 )
 
 func TestLazyTruncatedUserAgent(t *testing.T) {
@@ -71,3 +75,24 @@ func TestLazyAccept(t *testing.T) {
 	acceptWithoutReq := &lazyAccept{}
 	assert.Equal(t, "unknown", fmt.Sprintf("%v", acceptWithoutReq))
 }
+
+func TestLazyVerb(t *testing.T) {
+	assert.Equal(t, "unknown", fmt.Sprintf("%v", &lazyVerb{}))
+
+	u, _ := url.Parse("?watch=true")
+	req := &http.Request{Method: "GET", URL: u}
+	verbWithReq := &lazyVerb{req: req}
+	assert.Equal(t, "WATCH", fmt.Sprintf("%v", verbWithReq))
+}
+
+func TestLazyResource(t *testing.T) {
+	assert.Equal(t, "unknown", fmt.Sprintf("%v", &lazyResource{}))
+
+	resourceWithEmptyReq := &lazyResource{&http.Request{}}
+	assert.Equal(t, "unknown", fmt.Sprintf("%v", resourceWithEmptyReq))
+
+	req := &http.Request{}
+	ctx := request.WithRequestInfo(context.TODO(), &request.RequestInfo{Resource: "resource"})
+	resourceWithReq := &lazyResource{req: req.WithContext(ctx)}
+	assert.Equal(t, "resource", fmt.Sprintf("%v", resourceWithReq))
+}

pkg/endpoints/handlers/trace_util.go

@@ -24,11 +24,13 @@ import (
 
 func traceFields(req *http.Request) []attribute.KeyValue {
 	return []attribute.KeyValue{
-		attribute.String("url", req.URL.Path),
-		attribute.Stringer("user-agent", &lazyTruncatedUserAgent{req: req}),
+		attribute.Stringer("accept", &lazyAccept{req: req}),
 		attribute.Stringer("audit-id", &lazyAuditID{req: req}),
 		attribute.Stringer("client", &lazyClientIP{req: req}),
-		attribute.Stringer("accept", &lazyAccept{req: req}),
 		attribute.String("protocol", req.Proto),
+		attribute.Stringer("resource", &lazyResource{req: req}),
+		attribute.String("url", req.URL.Path),
+		attribute.Stringer("user-agent", &lazyTruncatedUserAgent{req: req}),
+		attribute.Stringer("verb", &lazyVerb{req: req}),
 	}
 }

pkg/endpoints/metrics/metrics.go

@@ -565,6 +565,20 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
 	}
 }
 
+// NormalizedVerb returns normalized verb
+func NormalizedVerb(req *http.Request) string {
+	verb := req.Method
+	if requestInfo, ok := request.RequestInfoFrom(req.Context()); ok {
+		// If we can find a requestInfo, we can get a scope, and then
+		// we can convert GETs to LISTs when needed.
+		scope := CleanScope(requestInfo)
+		verb = CanonicalVerb(strings.ToUpper(verb), scope)
+	}
+
+	// mark APPLY requests and WATCH requests correctly.
+	return CleanVerb(verb, req)
+}
+
 // CleanScope returns the scope of the request.
 func CleanScope(requestInfo *request.RequestInfo) string {
 	if requestInfo.Name != "" || requestInfo.Verb == "create" {

pkg/server/httplog/httplog.go

@@ -243,16 +243,7 @@ func SetStacktracePredicate(ctx context.Context, pred StacktracePred) {
 func (rl *respLogger) Log() {
 	latency := time.Since(rl.startTime)
 	auditID := audit.GetAuditIDTruncated(rl.req.Context())
-
-	verb := rl.req.Method
-	if requestInfo, ok := request.RequestInfoFrom(rl.req.Context()); ok {
-		// If we can find a requestInfo, we can get a scope, and then
-		// we can convert GETs to LISTs when needed.
-		scope := metrics.CleanScope(requestInfo)
-		verb = metrics.CanonicalVerb(strings.ToUpper(verb), scope)
-	}
-	// mark APPLY requests and WATCH requests correctly.
-	verb = metrics.CleanVerb(verb, rl.req)
+	verb := metrics.NormalizedVerb(rl.req)
 
 	keysAndValues := []interface{}{
 		"verb", verb,