Enable test retry for daily builds (#13086)
This commit is contained in:
Lauri Tulmin
GitHub
parent
834aea2406
commit
8aa2584094
|
|
@ -75,6 +75,7 @@ for [`dependabot/**/**`](https://github.com/open-telemetry/community/blob/main/d
|
|||
- Key is associated with [@trask](https://github.com/trask)'s gmail address
|
||||
- `SONATYPE_KEY` - owned by [@trask](https://github.com/trask)
|
||||
- `SONATYPE_USER` - owned by [@trask](https://github.com/trask)
|
||||
- `FLAKY_TEST_REPORTER_ACCESS_KEY` - owned by [@laurit](https://github.com/laurit)
|
||||
|
||||
### Organization secrets
|
||||
|
||||
|
|
|
|||
|
|
@ -290,6 +290,36 @@ jobs:
|
|||
if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
|
||||
run: cat build-scan.txt
|
||||
|
||||
- name: Get current job url
|
||||
id: jobs
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
matrix: ${{ toJson(matrix) }}
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100
|
||||
});
|
||||
const matrix = JSON.parse(process.env.matrix);
|
||||
const job_name = `common / test${ matrix['test-partition'] } (${ matrix['test-java-version'] }, ${ matrix.vm })`;
|
||||
return workflow_run.jobs.find((job) => job.name === job_name).html_url;
|
||||
|
||||
- name: Flaky test report
|
||||
if: ${{ !cancelled() }}
|
||||
env:
|
||||
FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
|
||||
JOB_URL: ${{ steps.jobs.outputs.result }}
|
||||
run: |
|
||||
if [ -s build-scan.txt ]; then
|
||||
export BUILD_SCAN_URL=$(cat build-scan.txt)
|
||||
fi
|
||||
./gradlew :test-report:reportFlakyTests
|
||||
|
||||
- name: Upload deadlock detector artifacts if any
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
|
|
|
|||
|
|
@ -10,19 +10,16 @@ jobs:
|
|||
common:
|
||||
uses: ./.github/workflows/build-common.yml
|
||||
with:
|
||||
max-test-retries: 0
|
||||
no-build-cache: true
|
||||
|
||||
test-latest-deps:
|
||||
uses: ./.github/workflows/reusable-test-latest-deps.yml
|
||||
with:
|
||||
max-test-retries: 0
|
||||
no-build-cache: true
|
||||
|
||||
test-indy:
|
||||
uses: ./.github/workflows/reusable-test-indy.yml
|
||||
with:
|
||||
max-test-retries: 0
|
||||
no-build-cache: true
|
||||
|
||||
# muzzle is not included here because it doesn't use gradle cache anyway and so is already covered
|
||||
|
|
|
|||
|
|
@ -9,18 +9,12 @@ on:
|
|||
jobs:
|
||||
common:
|
||||
uses: ./.github/workflows/build-common.yml
|
||||
with:
|
||||
max-test-retries: 0
|
||||
|
||||
test-latest-deps:
|
||||
uses: ./.github/workflows/reusable-test-latest-deps.yml
|
||||
with:
|
||||
max-test-retries: 0
|
||||
|
||||
test-indy:
|
||||
uses: ./.github/workflows/reusable-test-indy.yml
|
||||
with:
|
||||
max-test-retries: 0
|
||||
|
||||
muzzle:
|
||||
uses: ./.github/workflows/reusable-muzzle.yml
|
||||
|
|
|
|||
|
|
@ -86,3 +86,33 @@ jobs:
|
|||
- name: Build scan
|
||||
if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
|
||||
run: cat build-scan.txt
|
||||
|
||||
- name: Get current job url
|
||||
id: jobs
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
matrix: ${{ toJson(matrix) }}
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100
|
||||
});
|
||||
const matrix = JSON.parse(process.env.matrix);
|
||||
const job_name = `test-indy / testIndy${ matrix['test-partition'] }`;
|
||||
return workflow_run.jobs.find((job) => job.name === job_name).html_url;
|
||||
|
||||
- name: Flaky test report
|
||||
if: ${{ !cancelled() }}
|
||||
env:
|
||||
FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
|
||||
JOB_URL: ${{ steps.jobs.outputs.result }}
|
||||
run: |
|
||||
if [ -s build-scan.txt ]; then
|
||||
export BUILD_SCAN_URL=$(cat build-scan.txt)
|
||||
fi
|
||||
./gradlew :test-report:reportFlakyTests
|
||||
|
|
|
|||
|
|
@ -85,6 +85,36 @@ jobs:
|
|||
if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
|
||||
run: cat build-scan.txt
|
||||
|
||||
- name: Get current job url
|
||||
id: jobs
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
matrix: ${{ toJson(matrix) }}
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100
|
||||
});
|
||||
const matrix = JSON.parse(process.env.matrix);
|
||||
const job_name = `test-latest-deps / testLatestDeps${ matrix['test-partition'] }`;
|
||||
return workflow_run.jobs.find((job) => job.name === job_name).html_url;
|
||||
|
||||
- name: Flaky test report
|
||||
if: ${{ !cancelled() }}
|
||||
env:
|
||||
FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
|
||||
JOB_URL: ${{ steps.jobs.outputs.result }}
|
||||
run: |
|
||||
if [ -s build-scan.txt ]; then
|
||||
export BUILD_SCAN_URL=$(cat build-scan.txt)
|
||||
fi
|
||||
./gradlew :test-report:reportFlakyTests
|
||||
|
||||
- name: Upload deadlock detector artifacts if any
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
|
|
|
|||
|
|
@ -54,7 +54,8 @@ develocity {
|
|||
termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use")
|
||||
termsOfUseAgree.set("yes")
|
||||
|
||||
if (!gradle.startParameter.taskNames.contains("listTestsInPartition")) {
|
||||
if (!gradle.startParameter.taskNames.contains("listTestsInPartition") &&
|
||||
!gradle.startParameter.taskNames.contains(":test-report:reportFlakyTests")) {
|
||||
buildScanPublished {
|
||||
File("build-scan.txt").printWriter().use { writer ->
|
||||
writer.println(buildScanUri)
|
||||
|
|
@ -97,6 +98,7 @@ include(":instrumentation-annotations-support-testing")
|
|||
|
||||
// misc
|
||||
include(":dependencyManagement")
|
||||
include(":test-report")
|
||||
include(":testing:agent-exporter")
|
||||
include(":testing:agent-for-testing")
|
||||
include(":testing:armeria-shaded-for-testing")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
plugins {
|
||||
id("otel.java-conventions")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("com.google.api-client:google-api-client:2.7.1")
|
||||
implementation("com.google.apis:google-api-services-sheets:v4-rev20250106-2.0.0")
|
||||
implementation("com.google.auth:google-auth-library-oauth2-http:1.30.1")
|
||||
}
|
||||
|
||||
otelJava {
|
||||
minJavaVersionSupported.set(JavaVersion.VERSION_17)
|
||||
}
|
||||
|
||||
tasks {
|
||||
val reportFlakyTests by registering(JavaExec::class) {
|
||||
dependsOn(classes)
|
||||
|
||||
mainClass.set("io.opentelemetry.instrumentation.testreport.FlakyTestReporter")
|
||||
classpath(sourceSets["main"].runtimeClasspath)
|
||||
|
||||
systemProperty("scanPath", project.rootDir)
|
||||
systemProperty("googleSheetsAccessKey", System.getenv("FLAKY_TEST_REPORTER_ACCESS_KEY"))
|
||||
systemProperty("buildScanUrl", System.getenv("BUILD_SCAN_URL"))
|
||||
systemProperty("jobUrl", System.getenv("JOB_URL"))
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,282 @@
|
|||
/*
|
||||
* Copyright The OpenTelemetry Authors
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package io.opentelemetry.instrumentation.testreport;
|
||||
|
||||
import static java.nio.file.FileVisitResult.CONTINUE;
|
||||
|
||||
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
|
||||
import com.google.api.client.http.javanet.NetHttpTransport;
|
||||
import com.google.api.client.json.gson.GsonFactory;
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.SheetsScopes;
|
||||
import com.google.api.services.sheets.v4.model.ValueRange;
|
||||
import com.google.auth.http.HttpCredentialsAdapter;
|
||||
import com.google.auth.oauth2.GoogleCredentials;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.NamedNodeMap;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
@SuppressWarnings("SystemOut")
|
||||
public class FlakyTestReporter {
|
||||
// https://docs.google.com/spreadsheets/d/1pfa6Ws980AIFI3kKOeIc51-JGEzakG7hkMl4J9h-Tk0
|
||||
private static final String SPREADSHEET_ID = "1pfa6Ws980AIFI3kKOeIc51-JGEzakG7hkMl4J9h-Tk0";
|
||||
|
||||
private int testCount;
|
||||
private int skippedCount;
|
||||
private int failureCount;
|
||||
private int errorCount;
|
||||
private final List<FlakyTest> flakyTests = new ArrayList<>();
|
||||
|
||||
private static class FlakyTest {
|
||||
final String testClassName;
|
||||
final String testName;
|
||||
final String timestamp;
|
||||
final String message;
|
||||
|
||||
FlakyTest(String testClassName, String testName, String timestamp, String message) {
|
||||
this.testClassName = testClassName;
|
||||
this.testName = testName;
|
||||
this.timestamp = timestamp;
|
||||
this.message = message;
|
||||
}
|
||||
}
|
||||
|
||||
private void addFlakyTest(
|
||||
String testClassName, String testName, String timestamp, String message) {
|
||||
flakyTests.add(new FlakyTest(testClassName, testName, timestamp, message));
|
||||
}
|
||||
|
||||
private static Document parse(Path testReport) {
|
||||
try {
|
||||
DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
|
||||
return builder.parse(testReport.toFile());
|
||||
} catch (Exception exception) {
|
||||
throw new IllegalStateException("failed to parse test report " + testReport, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private void scanTestFile(Path testReport) {
|
||||
Document doc = parse(testReport);
|
||||
doc.getDocumentElement().normalize();
|
||||
testCount += Integer.parseInt(doc.getDocumentElement().getAttribute("tests"));
|
||||
skippedCount += Integer.parseInt(doc.getDocumentElement().getAttribute("skipped"));
|
||||
int failures = Integer.parseInt(doc.getDocumentElement().getAttribute("failures"));
|
||||
failureCount += failures;
|
||||
int errors = Integer.parseInt(doc.getDocumentElement().getAttribute("errors"));
|
||||
errorCount += errors;
|
||||
String timestamp = doc.getDocumentElement().getAttribute("timestamp");
|
||||
|
||||
// there are no flaky tests if there are no failures, skip it
|
||||
if (failures == 0 && errors == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
class TestCase {
|
||||
final String className;
|
||||
final String name;
|
||||
boolean failed;
|
||||
boolean succeeded;
|
||||
String message;
|
||||
|
||||
TestCase(String className, String name) {
|
||||
this.className = className;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
boolean isFlaky() {
|
||||
return succeeded && failed;
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, TestCase> testcaseMap = new HashMap<>();
|
||||
|
||||
NodeList testcaseNodes = doc.getElementsByTagName("testcase");
|
||||
for (int i = 0; i < testcaseNodes.getLength(); i++) {
|
||||
Node testNode = testcaseNodes.item(i);
|
||||
|
||||
String testClassName = testNode.getAttributes().getNamedItem("classname").getNodeValue();
|
||||
String testName = testNode.getAttributes().getNamedItem("name").getNodeValue();
|
||||
String testKey = testClassName + "." + testName;
|
||||
TestCase testCase =
|
||||
testcaseMap.computeIfAbsent(testKey, (s) -> new TestCase(testClassName, testName));
|
||||
NodeList childNodes = testNode.getChildNodes();
|
||||
boolean failed = false;
|
||||
for (int j = 0; j < childNodes.getLength(); j++) {
|
||||
Node childNode = childNodes.item(j);
|
||||
String nodeName = childNode.getNodeName();
|
||||
if ("failure".equals(nodeName) || "error".equals(nodeName)) {
|
||||
failed = true;
|
||||
// if test fails multiple times we'll use the first failure message
|
||||
if (testCase.message == null) {
|
||||
String message = getAttributeValue(childNode, "message");
|
||||
if (message != null) {
|
||||
// compress failure message on a single line
|
||||
message = message.replaceAll("\n( )*", " ");
|
||||
}
|
||||
testCase.message = message;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failed) {
|
||||
testCase.failed = true;
|
||||
} else {
|
||||
testCase.succeeded = true;
|
||||
}
|
||||
}
|
||||
|
||||
for (TestCase testCase : testcaseMap.values()) {
|
||||
if (testCase.isFlaky()) {
|
||||
addFlakyTest(testCase.className, testCase.name, timestamp, testCase.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String getAttributeValue(Node node, String attributeName) {
|
||||
NamedNodeMap attributes = node.getAttributes();
|
||||
if (attributes == null) {
|
||||
return null;
|
||||
}
|
||||
Node value = attributes.getNamedItem(attributeName);
|
||||
return value != null ? value.getNodeValue() : null;
|
||||
}
|
||||
|
||||
private void scanTestResults(Path buildDir) throws IOException {
|
||||
Path testResults = buildDir.resolve("test-results");
|
||||
if (!Files.exists(testResults)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Files.walkFileTree(
|
||||
testResults,
|
||||
new SimpleFileVisitor<>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
String name = file.getFileName().toString();
|
||||
if (name.startsWith("TEST-") && name.endsWith(".xml")) {
|
||||
scanTestFile(file);
|
||||
}
|
||||
|
||||
return CONTINUE;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static FlakyTestReporter scan(Path path) throws IOException {
|
||||
FlakyTestReporter reporter = new FlakyTestReporter();
|
||||
Files.walkFileTree(
|
||||
path,
|
||||
new SimpleFileVisitor<>() {
|
||||
@Override
|
||||
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
|
||||
throws IOException {
|
||||
if (dir.endsWith("build")) {
|
||||
reporter.scanTestResults(dir);
|
||||
return FileVisitResult.SKIP_SUBTREE;
|
||||
}
|
||||
if (dir.endsWith("src")) {
|
||||
return FileVisitResult.SKIP_SUBTREE;
|
||||
}
|
||||
return CONTINUE;
|
||||
}
|
||||
});
|
||||
return reporter;
|
||||
}
|
||||
|
||||
private void print() {
|
||||
System.err.printf(
|
||||
"Found %d test, skipped %d, failed %d, errored %d\n",
|
||||
testCount, skippedCount, failureCount, errorCount);
|
||||
if (!flakyTests.isEmpty()) {
|
||||
System.err.printf("Found %d flaky test(s):\n", flakyTests.size());
|
||||
for (FlakyTest flakyTest : flakyTests) {
|
||||
System.err.println(
|
||||
flakyTest.timestamp
|
||||
+ " "
|
||||
+ flakyTest.testClassName
|
||||
+ " "
|
||||
+ flakyTest.testName
|
||||
+ " "
|
||||
+ flakyTest.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add flaky tests to a google sheet
|
||||
private void report(String accessKey, String buildScanUrl, String jobUrl) throws Exception {
|
||||
if (flakyTests.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
|
||||
GoogleCredentials credentials =
|
||||
GoogleCredentials.fromStream(
|
||||
new ByteArrayInputStream(accessKey.getBytes(StandardCharsets.UTF_8)))
|
||||
.createScoped(Collections.singletonList(SheetsScopes.SPREADSHEETS));
|
||||
Sheets service =
|
||||
new Sheets.Builder(
|
||||
transport,
|
||||
GsonFactory.getDefaultInstance(),
|
||||
new HttpCredentialsAdapter(credentials))
|
||||
.setApplicationName("Flaky test reporter")
|
||||
.build();
|
||||
|
||||
List<List<Object>> data = new ArrayList<>();
|
||||
for (FlakyTest flakyTest : flakyTests) {
|
||||
List<Object> row = new ArrayList<>();
|
||||
row.add(flakyTest.timestamp);
|
||||
row.add(flakyTest.testClassName);
|
||||
row.add(flakyTest.testName);
|
||||
row.add(buildScanUrl);
|
||||
row.add(jobUrl);
|
||||
row.add(flakyTest.message);
|
||||
data.add(row);
|
||||
}
|
||||
|
||||
ValueRange valueRange = new ValueRange();
|
||||
valueRange.setValues(data);
|
||||
service
|
||||
.spreadsheets()
|
||||
.values()
|
||||
.append(SPREADSHEET_ID, "Sheet1!A:F", valueRange)
|
||||
.setValueInputOption("USER_ENTERED")
|
||||
.execute();
|
||||
}
|
||||
|
||||
public static void main(String... args) throws Exception {
|
||||
String path = System.getProperty("scanPath");
|
||||
if (path == null) {
|
||||
throw new IllegalStateException("scanPath system property must be set");
|
||||
}
|
||||
File file = new File(path).getAbsoluteFile();
|
||||
System.err.println("Scanning for flaky tests in " + file.getPath());
|
||||
FlakyTestReporter reporter = FlakyTestReporter.scan(file.toPath());
|
||||
reporter.print();
|
||||
|
||||
String accessKey = System.getProperty("googleSheetsAccessKey");
|
||||
String buildScanUrl = System.getProperty("buildScanUrl");
|
||||
String jobUrl = System.getProperty("jobUrl");
|
||||
if (accessKey != null && !accessKey.isEmpty()) {
|
||||
reporter.report(accessKey, buildScanUrl, jobUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue