test/webhook-apicoverage: Add support to produce junit xml results (#651)

This change adds support to webhook apicoverage to produce coverage
results in junit xml format. The change only produces coverage
percentages. This can be used to push the results to end points like
test-grid.

The change also moves all the template files as go-strings as finding
relative paths to these files becomes tricky based on the deployment
environment.

test/webhook-apicoverage/coveragecalculator/calculator.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package coveragecalculator
 
+import (
+	"math"
+)
+
 // CoverageValues encapsulates all the coverage related values.
 type CoverageValues struct {
 	TotalFields   int
@@ -25,12 +29,37 @@ type CoverageValues struct {
 	PercentCoverage float64
 }
 
+// CoveragePercentages encapsulate percentage coverage for resources.
+type CoveragePercentages struct {
+
+	// ResourceCoverages maps percentage coverage per resource.
+	ResourceCoverages map[string]float64
+}
+
+// CalculatePercentageValue calculates percentage value based on other fields.
 func (c *CoverageValues) CalculatePercentageValue() {
 	if c.TotalFields > 0 {
 		c.PercentCoverage = (float64(c.CoveredFields) / float64(c.TotalFields-c.IgnoredFields)) * 100
 	}
 }
 
+// GetAndRemoveResourceValue utility method to implement "get and delete"
+// semantics. This makes templating operations easy.
+func (c *CoveragePercentages) GetAndRemoveResourceValue(resource string) float64 {
+	if resourcePercentage, ok := c.ResourceCoverages[resource]; ok {
+		delete(c.ResourceCoverages, resource)
+		return resourcePercentage
+	}
+
+	return 0.0
+}
+
+// IsFailedBuild utility method to indicate if CoveragePercentages indicate
+// values of a failed build.
+func (c *CoveragePercentages) IsFailedBuild() bool {
+	return math.Abs(c.ResourceCoverages["Overall"] - 0) == 0
+}
+
 // CalculateTypeCoverage calculates aggregate coverage values based on provided []TypeCoverage
 func CalculateTypeCoverage(typeCoverage []TypeCoverage) *CoverageValues {
 	cv := CoverageValues{}

test/webhook-apicoverage/tools/tools.go

@@ -25,6 +25,7 @@ import (
 	"os/user"
 	"path"
 
+	"github.com/pkg/errors"
 	"knative.dev/pkg/test/webhook-apicoverage/coveragecalculator"
 	"knative.dev/pkg/test/webhook-apicoverage/view"
 	"knative.dev/pkg/test/webhook-apicoverage/webhook"
@@ -47,6 +48,10 @@ const (
 
 	// WebhookTotalCoverageEndPoint constant for total coverage API endpoint.
 	WebhookTotalCoverageEndPoint = "https://%s:443" + webhook.TotalCoverageEndPoint
+
+	// WebhookResourcePercentageCoverageEndPoint constant for
+	// ResourcePercentageCoverage API endpoint.
+	WebhookResourcePercentageCoverageEndPoint = "https://%s:443" + webhook.ResourcePercentageCoverageEndPoint
 )
 
 // GetDefaultKubePath helper method to fetch kubeconfig path.
@@ -107,14 +112,15 @@ func GetResourceCoverage(webhookIP string, resourceName string) (string, error)
 	}
 	resp, err := client.Get(fmt.Sprintf(WebhookResourceCoverageEndPoint, webhookIP, resourceName))
 	if err != nil {
-		return "", fmt.Errorf("encountered error making resource coverage request: %v", err)
-	} else if resp.StatusCode != http.StatusOK {
+		return "", errors.Wrap(err, "encountered error making resource coverage request")
+	}
+	if resp.StatusCode != http.StatusOK {
 		return "", fmt.Errorf("invalid HTTP Status received for resource coverage request. Status: %d", resp.StatusCode)
 	}
 
 	var body []byte
 	if body, err = ioutil.ReadAll(resp.Body); err != nil {
-		return "", fmt.Errorf("error reading resource coverage response: %v", err)
+		return "", errors.Wrap(err, "Failed reading resource coverage response")
 	}
 
 	return string(body), nil
@@ -131,11 +137,7 @@ func GetAndWriteResourceCoverage(webhookIP string, resourceName string, outputFi
 		return err
 	}
 
-	if err = ioutil.WriteFile(outputFile, []byte(resourceCoverage), 0400); err != nil {
-		return fmt.Errorf("error writing resource coverage to output file: %s, error: %v coverage: %s", outputFile, err, resourceCoverage)
-	}
-
-	return nil
+	return ioutil.WriteFile(outputFile, []byte(resourceCoverage), 0400)
 }
 
 // GetTotalCoverage calls the total coverage API to retrieve total coverage values.
@@ -147,8 +149,9 @@ func GetTotalCoverage(webhookIP string) (*coveragecalculator.CoverageValues, err
 
 	resp, err := client.Get(fmt.Sprintf(WebhookTotalCoverageEndPoint, webhookIP))
 	if err != nil {
-		return nil, fmt.Errorf("encountered error making total coverage request: %v", err)
-	} else if resp.StatusCode != http.StatusOK {
+		return nil, errors.Wrap(err, "encountered error making total coverage request")
+	}
+	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("invalid HTTP Status received for total coverage request. Status: %d", resp.StatusCode)
 	}
 
@@ -159,7 +162,7 @@ func GetTotalCoverage(webhookIP string) (*coveragecalculator.CoverageValues, err
 
 	var coverage coveragecalculator.CoverageValues
 	if err = json.Unmarshal(body, &coverage); err != nil {
-		return nil, fmt.Errorf("error unmarshalling response to CoverageValues instance: %v", err)
+		return nil, errors.Wrap(err, "Failed unmarshalling response to CoverageValues instance")
 	}
 
 	return &coverage, nil
@@ -176,13 +179,53 @@ func GetAndWriteTotalCoverage(webhookIP string, outputFile string) error {
 		return err
 	}
 
-	if htmlData, err := view.GetHTMLCoverageValuesDisplay(totalCoverage); err != nil {
-		return fmt.Errorf("error building html file from total coverage. error: %v", err)
-	} else {
-		if err = ioutil.WriteFile(outputFile, []byte(htmlData), 0400); err != nil {
-			return fmt.Errorf("error writing total coverage to output file: %s, error: %v", outputFile, err)
-		}
+	htmlData, err := view.GetHTMLCoverageValuesDisplay(totalCoverage)
+	if err != nil {
+		return errors.Wrap(err, "Failed building html file from total coverage. error")
 	}
 
-	return nil
+	return ioutil.WriteFile(outputFile, []byte(htmlData), 0400)
+}
+
+// GetResourcePercentages calls resource percentage coverage API to retrieve
+// percentage values.
+func GetResourcePercentages(webhookIP string) (
+	*coveragecalculator.CoveragePercentages, error) {
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	},
+	}
+
+	resp, err := client.Get(fmt.Sprintf(WebhookResourcePercentageCoverageEndPoint,
+		webhookIP))
+	if err != nil {
+		return nil, errors.Wrap(err, "encountered error making resource percentage coverage request")
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("Invalid HTTP Status received for resource"+
+			" percentage coverage request. Status: %d", resp.StatusCode)
+	}
+
+	var body []byte
+	if body, err = ioutil.ReadAll(resp.Body); err != nil {
+		return nil, errors.Wrap(err, "Failed reading resource percentage coverage response")
+	}
+
+	coveragePercentages := &coveragecalculator.CoveragePercentages{}
+	if err = json.Unmarshal(body, coveragePercentages); err != nil {
+		return nil, errors.Wrap(err, "Failed unmarshalling response to CoveragePercentages instance")
+	}
+
+	return coveragePercentages, nil
+}
+
+// WriteResourcePercentages writes CoveragePercentages to junit_xml output file.
+func WriteResourcePercentages(outputFile string,
+	coveragePercentages *coveragecalculator.CoveragePercentages) error {
+	htmlData, err := view.GetCoveragePercentageXMLDisplay(coveragePercentages)
+	if err != nil {
+		errors.Wrap(err, "Failed building coverage percentage xml file")
+	}
+
+	return ioutil.WriteFile(outputFile, []byte(htmlData), 0400)
 }

test/webhook-apicoverage/view/README.md

@@ -37,3 +37,8 @@ Covered Fields: <Number of fields covered>
 Ignored Fields: <Number of fields ignored>
 Coverage Percentage: <Percentage value of coverage>
 ```
+
+`GetCoveragePercentageXMLDisplay()` is a utility method that can be used by
+repos to produce coverage percentage for each resource in a Junit XML results
+file. The method takes [CoveragePercentages](../coveragecalculator/calculator.go)
+as input and produces a Junit result file format.

test/webhook-apicoverage/view/aggregate_coverage.html

@@ -1,25 +0,0 @@
-<!DOCTYPE html>
-<html>
-<style type="text/css">
-  <!--
-
-  .tab { margin-left: 50px; }
-
-  .styleheader {color: white; size: A4}
-
-  .values {color: yellow; size: A3}
-
-  table, th, td { border: 1px solid white; text-align: center}
-
-  .braces {color: white; size: A3}
-  -->
-</style>
-<body style="background-color:rgb(0,0,0); font-family: Arial">
-<table style="width: 30%">
-  <tr class="styleheader"><td>Total Fields</td><td>{{ .TotalFields }}</td></tr>
-  <tr class="styleheader"><td>Covered Fields</td><td>{{ .CoveredFields }}</td></tr>
-  <tr class="styleheader"><td>Ignored Fields</td><td>{{ .IgnoredFields }}</td></tr>
-  <tr class="styleheader"><td>Coverage Percentage</td><td>{{ .PercentCoverage }}</td></tr>
-</table>
-</body>
-</html>

test/webhook-apicoverage/view/html_display.go

@@ -28,15 +28,16 @@ type HtmlDisplayData struct {
 	CoverageNumbers *coveragecalculator.CoverageValues
 }
 
-// GetHTMLDisplay is a helper method to display API Coverage details in json-like format inside a HTML page.
-func GetHTMLDisplay(coverageData []coveragecalculator.TypeCoverage, coverageValues *coveragecalculator.CoverageValues) (string, error) {
-
+// GetHTMLDisplay is a helper method to display API Coverage details in
+// json-like format inside a HTML page.
+func GetHTMLDisplay(coverageData []coveragecalculator.TypeCoverage,
+	coverageValues *coveragecalculator.CoverageValues) (string, error) {
 	htmlData := HtmlDisplayData{
 		TypeCoverages:   coverageData,
 		CoverageNumbers: coverageValues,
 	}
 
-	tmpl, err := template.ParseFiles("type_coverage.html")
+	tmpl, err := template.New("TypeCoverage").Parse(TypeCoverageTempl)
 	if err != nil {
 		return "", err
 	}
@@ -52,8 +53,7 @@ func GetHTMLDisplay(coverageData []coveragecalculator.TypeCoverage, coverageValu
 
 // GetHTMLCoverageValuesDisplay is a helper method to display coverage values inside a HTML table.
 func GetHTMLCoverageValuesDisplay(coverageValues *coveragecalculator.CoverageValues) (string, error) {
-
-	tmpl, err := template.ParseFiles("aggregate_coverage.html")
+	tmpl, err := template.New("AggregateCoverage").Parse(AggregateCoverageTmpl)
 	if err != nil {
 		return "", err
 	}

test/webhook-apicoverage/view/html_template.go

@@ -1,4 +1,26 @@
-<!DOCTYPE html>
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package view
+
+import (
+	"fmt"
+)
+
+var TypeCoverageTempl = fmt.Sprint(`<!DOCTYPE html>
 <html>
 <style type="text/css">
   <!--
@@ -59,3 +81,31 @@
 </table>
 </body>
 </html>
+`)
+
+var AggregateCoverageTmpl = fmt.Sprint(`<!DOCTYPE html>
+<html>
+<style type="text/css">
+  <!--
+
+  .tab { margin-left: 50px; }
+
+  .styleheader {color: white; size: A4}
+
+  .values {color: yellow; size: A3}
+
+  table, th, td { border: 1px solid white; text-align: center}
+
+  .braces {color: white; size: A3}
+  -->
+</style>
+<body style="background-color:rgb(0,0,0); font-family: Arial">
+<table style="width: 30%">
+  <tr class="styleheader"><td>Total Fields</td><td>{{ .TotalFields }}</td></tr>
+  <tr class="styleheader"><td>Covered Fields</td><td>{{ .CoveredFields }}</td></tr>
+  <tr class="styleheader"><td>Ignored Fields</td><td>{{ .IgnoredFields }}</td></tr>
+  <tr class="styleheader"><td>Coverage Percentage</td><td>{{ .PercentCoverage }}</td></tr>
+</table>
+</body>
+</html>
+`)

test/webhook-apicoverage/view/xml_display.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package view
+
+import (
+	"strings"
+	"text/template"
+
+	"knative.dev/pkg/test/webhook-apicoverage/coveragecalculator"
+)
+
+// GetCoveragePercentageXMLDisplay is a helper method to write resource coverage
+// percentage values to junit xml file format.
+func GetCoveragePercentageXMLDisplay(
+		percentageCoverages *coveragecalculator.CoveragePercentages) (string, error) {
+	tmpl, err := template.New("JunitResult").Parse(JunitResultTmpl)
+	if err != nil {
+		return "", err
+	}
+
+	var buffer strings.Builder
+	err = tmpl.Execute(&buffer, percentageCoverages)
+	if err != nil {
+		return "", err
+	}
+
+	return buffer.String(), nil
+}

test/webhook-apicoverage/view/xml_template.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package view
+
+import (
+	"fmt"
+)
+
+var JunitResultTmpl = fmt.Sprint(`<testsuites>
+  <testsuite name="" time="0" {{ if .IsFailedBuild }} failures="1" {{ else }} failures = "0" {{ end }} tests="0">
+      <testcase name="Overall" time="0" classname="go_coverage">
+				{{ if .IsFailedBuild }}
+					<failure>true</failure>
+        {{ end }}
+				<properties>
+          <property name="coverage" value="{{ .GetAndRemoveResourceValue "Overall" }}"/>
+        </properties>
+      </testcase>
+    {{ range $key, $value := .ResourceCoverages }}
+      <testcase name="{{ $key }}" time="0" classname="go_coverage">
+        <properties>
+          <property name="coverage" value="{{ $value }}"/>
+        </properties>
+      </testcase>
+    {{end}}
+  </testsuite>
+</testsuites>`)

test/webhook-apicoverage/webhook/apicoverage_recorder.go

@@ -50,6 +50,10 @@ const (
 	// TotalCoverageEndPoint is the endpoint for Total Coverage API
 	TotalCoverageEndPoint = "/totalcoverage"
 
+	// ResourcePercentageCoverageEndPoint is the end point for Resource Percentage
+	// coverages API
+	ResourcePercentageCoverageEndPoint = "/resourcepercentagecoverage"
+
 	// resourceChannelQueueSize size of the queue maintained for resource channel.
 	resourceChannelQueueSize = 10
 )
@@ -209,3 +213,51 @@ func (a *APICoverageRecorder) GetTotalCoverage(w http.ResponseWriter, r *http.Re
 		fmt.Fprintf(w, "error writing total coverage response: %v", err)
 	}
 }
+
+// GetResourceCoveragePercentags goes over all the resources setup for the
+// apicoverage tool and returns percentage coverage for each resource.
+func (a *APICoverageRecorder) GetResourceCoveragePercentages(
+	w http.ResponseWriter, r *http.Request) {
+	var (
+		ignoredFields coveragecalculator.IgnoredFields
+		err           error
+	)
+
+	ignoredFieldsFilePath :=
+		os.Getenv("KO_DATA_PATH") + "/ignoredfields.yaml"
+	if err = ignoredFields.ReadFromFile(ignoredFieldsFilePath); err != nil {
+		a.Logger.Errorf("Error reading file %s: %v",
+			ignoredFieldsFilePath, err)
+	}
+
+	totalCoverage := coveragecalculator.CoverageValues{}
+	percentCoverages := make(map[string]float64)
+	for resource := range a.ResourceMap {
+		tree := a.ResourceForest.TopLevelTrees[resource.Kind]
+		typeCoverage := tree.BuildCoverageData(a.NodeRules, a.FieldRules,
+			ignoredFields)
+		coverageValues := coveragecalculator.CalculateTypeCoverage(typeCoverage)
+		coverageValues.CalculatePercentageValue()
+		percentCoverages[resource.Kind] = coverageValues.PercentCoverage
+		totalCoverage.TotalFields += coverageValues.TotalFields
+		totalCoverage.CoveredFields += coverageValues.CoveredFields
+		totalCoverage.IgnoredFields += coverageValues.IgnoredFields
+	}
+	totalCoverage.CalculatePercentageValue()
+	percentCoverages["Overall"] = totalCoverage.PercentCoverage
+
+	var body []byte
+	if body, err = json.Marshal(
+		coveragecalculator.CoveragePercentages{
+			ResourceCoverages: percentCoverages,
+		}); err != nil {
+		fmt.Fprintf(w, "error marshalling percentage coverage response: %v",
+			err)
+		return
+	}
+
+	if _, err = w.Write(body); err != nil {
+		fmt.Fprintf(w, "error writing percentage coverage response: %v",
+			err)
+	}
+}