kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(frontend): upgrade frontend integration test dependencies and base image. (#8806) 

* upgrade frontend integration test dependencies and base image.

* remote unused libraries

* fix junit config

* fix junit config
This commit is contained in:
Chen Sun 2023-02-06 17:24:39 -08:00 committed by GitHub
parent 6522e6c48a
commit fe71e17a4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 5311 additions and 1561 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/frontend-integration-test/Dockerfile
View File

@ -1,4 +1,4 @@
FROM gcr.io/ml-pipeline-test/selenium-standalone-chrome-gcloud-nodejs:v20200210-0.2.2-30-g05865480-e3b0c4 FROM gcr.io/ml-pipeline-test/selenium-standalone-chrome-gcloud-nodejs:v20230201-2.0.0b11-74-g395719cab-dirty-9bed65
#To build this image: cd selenium-standalone-chrome-gcloud-nodejs.Docker && make push #To build this image: cd selenium-standalone-chrome-gcloud-nodejs.Docker && make push
COPY --chown=seluser . /src COPY --chown=seluser . /src

308
test/frontend-integration-test/helloworld.spec.js
View File

@ -1,4 +1,4 @@
// Copyright 2018 The Kubeflow Authors // Copyright 2018-2023 The Kubeflow Authors
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -21,272 +21,292 @@ const pipelineName = 'helloworld-pipeline-' + Date.now();
const runName = 'helloworld-' + Date.now(); const runName = 'helloworld-' + Date.now();
const runDescription = 'test run description ' + runName; const runDescription = 'test run description ' + runName;
const runWithoutExperimentName = 'helloworld-2-' + Date.now(); const runWithoutExperimentName = 'helloworld-2-' + Date.now();
const runWithoutExperimentDescription = 'test run without experiment description ' + runWithoutExperimentName; const runWithoutExperimentDescription =
'test run without experiment description ' + runWithoutExperimentName;
const waitTimeout = 5000; const waitTimeout = 5000;
const outputParameterValue = 'Hello world in test' const outputParameterValue = 'Hello world in test';
function getValueFromDetailsTable(key) { async function getValueFromDetailsTable(key) {
// Find the span that shows the key, get its parent div (the row), then // Find the span that shows the key, get its parent div (the row), then
// get that row's inner text, and remove the key // get that row's inner text, and remove the key
const row = $(`span=${key}`).$('..'); const rowText = await $(`span=${key}`).$('..').getText();
return row.getText().substr(`${key}\n`.length); return rowText.substr(`${key}\n`.length);
} }
describe('deploy helloworld sample run', () => { describe('deploy helloworld sample run', () => {
before(async () => {
before(() => { await browser.url('/');
browser.url('/');
}); });
it('open pipeline creation page', () => { it('open pipeline creation page', async () => {
$('#createPipelineVersionBtn').click(); await $('#createPipelineVersionBtn').click();
browser.waitUntil(() => { await browser.waitUntil(async () => {
return new URL(browser.getUrl()).hash.startsWith('#/pipeline_versions/new'); return new URL(await browser.getUrl()).hash.startsWith('#/pipeline_versions/new');
}, waitTimeout); }, waitTimeout);
}); });
it('uploads the sample pipeline', () => { it('uploads the sample pipeline', async () => {
$('#localPackageBtn').click(); await $('#localPackageBtn').click();
browser.chooseFile('#dropZone input[type="file"]', './helloworld.yaml'); const remoteFilePath = await browser.uploadFile('./helloworld.yaml');
$('#newPipelineName').setValue(pipelineName); await $('#dropZone input[type="file"]').addValue(remoteFilePath);
$('#createNewPipelineOrVersionBtn').click(); await $('#newPipelineName').setValue(pipelineName);
browser.waitUntil(() => { await $('#createNewPipelineOrVersionBtn').click();
return new URL(browser.getUrl()).hash.startsWith('#/pipelines/details'); await browser.waitUntil(async () => {
return new URL(await browser.getUrl()).hash.startsWith('#/pipelines/details');
}, waitTimeout); }, waitTimeout);
}); });
it('shows a 4-node static graph', () => { it('shows a 4-node static graph', async () => {
const nodeSelector = '.graphNode'; const nodeSelector = '.graphNode';
$(nodeSelector).waitForVisible(); await $(nodeSelector);
const nodes = $$(nodeSelector).length; const nodes = await $$(nodeSelector);
assert(nodes === 4, 'should have a 4-node graph, instead has: ' + nodes); assert(nodes.length === 4, 'should have a 4-node graph, instead has: ' + nodes.length);
}); });
it('creates a new experiment out of this pipeline', () => { it('creates a new experiment out of this pipeline', async () => {
$('#newExperimentBtn').click(); await $('#newExperimentBtn').click();
browser.waitUntil(() => { await browser.waitUntil(async () => {
return new URL(browser.getUrl()).hash.startsWith('#/experiments/new'); return new URL(await browser.getUrl()).hash.startsWith('#/experiments/new');
}, waitTimeout); }, waitTimeout);
$('#experimentName').setValue(experimentName); await $('#experimentName').setValue(experimentName);
$('#experimentDescription').setValue(experimentDescription); await $('#experimentDescription').setValue(experimentDescription);
$('#createExperimentBtn').click(); await $('#createExperimentBtn').click();
}); });
it('creates a new run in the experiment', () => { it('creates a new run in the experiment', async () => {
$('#choosePipelineBtn').waitForVisible(); await $('#choosePipelineBtn').waitForDisplayed();
$('#choosePipelineBtn').click(); await $('#choosePipelineBtn').click();
$('.tableRow').waitForVisible(); await $('.tableRow').waitForDisplayed();
$('.tableRow').click(); await $('.tableRow').click();
$('#usePipelineBtn').click(); await $('#usePipelineBtn').click();
$('#pipelineSelectorDialog').waitForVisible(waitTimeout, true); await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: waitTimeout, reverse: true });
$('#choosePipelineVersionBtn').waitForVisible(); await $('#choosePipelineVersionBtn').waitForDisplayed();
$('#choosePipelineVersionBtn').click(); await $('#choosePipelineVersionBtn').click();
$('.tableRow').waitForVisible(); await $('.tableRow').waitForDisplayed();
$('.tableRow').click(); await $('.tableRow').click();
$('#usePipelineVersionBtn').click(); await $('#usePipelineVersionBtn').click();
$('#pipelineVersionSelectorDialog').waitForVisible(waitTimeout, true); await $('#pipelineVersionSelectorDialog').waitForDisplayed({
browser.pause(1000); timeout: waitTimeout,
browser.keys(runName); reverse: true,
});
browser.keys('Tab'); await browser.keys(runName);
browser.keys(runDescription);
await browser.keys('Tab');
await browser.keys(runDescription);
// Skip over "choose experiment" button // Skip over "choose experiment" button
browser.keys('Tab'); await browser.keys('Tab');
// Skip over service account help button // Skip over service account help button
browser.keys('Tab'); await browser.keys('Tab');
// Skip over "service account" textbox // Skip over "service account" textbox
browser.keys('Tab'); await browser.keys('Tab');
// Skip over "Run Type" radio button // Skip over "Run Type" radio button
browser.keys('Tab'); await browser.keys('Tab');
browser.keys('Tab'); await browser.keys('Tab');
browser.keys(outputParameterValue); await browser.keys(outputParameterValue);
// Deploy // Deploy
$('#startNewRunBtn').click(); await $('#startNewRunBtn').click();
}); });
it('redirects back to experiment page', () => { it('redirects back to experiment page', async () => {
browser.waitUntil(() => { await browser.waitUntil(async () => {
return new URL(browser.getUrl()).hash.startsWith('#/experiments/details/'); return new URL(await browser.getUrl()).hash.startsWith('#/experiments/details/');
}, waitTimeout); }, waitTimeout);
}); });
it('finds the new run in the list of runs, navigates to it', () => { it('finds the new run in the list of runs, navigates to it', async () => {
let attempts = 30; let attempts = 30;
// Wait for a reasonable amount of time until the run starts // Wait for a reasonable amount of time until the run starts
while (attempts && !$('.tableRow a').isExisting()) { while (attempts && !$('.tableRow a').isExisting()) {
browser.pause(1000); await browser.pause(1000);
$('#refreshBtn').click(); await $('#refreshBtn').click();
--attempts; --attempts;
} }
assert(attempts, 'waited for 30 seconds but run did not start.'); assert(attempts, 'waited for 30 seconds but run did not start.');
assert.equal($$('.tableRow').length, 1, 'should only show one run'); assert.equal(await $$('.tableRow').length, 1, 'should only show one run');
// Navigate to details of the deployed run by clicking its anchor element // Navigate to details of the deployed run by clicking its anchor element
browser.execute('document.querySelector(".tableRow a").click()'); await browser.execute('document.querySelector(".tableRow a").click()');
}); });
it('switches to config tab', () => { it('switches to config tab', async () => {
$('button=Config').waitForVisible(waitTimeout); await $('button=Config').waitForDisplayed({ timeout: waitTimeout });
$('button=Config').click(); await $('button=Config').click();
}); });
it('waits for run to finish', () => { it('waits for run to finish', async () => {
let status = getValueFromDetailsTable('Status'); let status = await getValueFromDetailsTable('Status');
let attempts = 0; let attempts = 0;
const maxAttempts = 60; const maxAttempts = 60;
// Wait for a reasonable amount of time until the run is done // Wait for a reasonable amount of time until the run is done
while (attempts < maxAttempts && status.trim() !== 'Succeeded') { while (attempts < maxAttempts && status.trim() !== 'Succeeded') {
browser.pause(1000); await browser.pause(1000);
status = getValueFromDetailsTable('Status'); status = await getValueFromDetailsTable('Status');
attempts++; attempts++;
} }
assert(attempts < maxAttempts, `waited for ${maxAttempts} seconds but run did not succeed. ` + assert(
'Current status is: ' + status); attempts < maxAttempts,
`waited for ${maxAttempts} seconds but run did not succeed. ` +
'Current status is: ' +
status,
);
}); });
it('displays run created at date correctly', () => { it('displays run created at date correctly', async () => {
const date = getValueFromDetailsTable('Created at'); const date = await getValueFromDetailsTable('Created at');
assert(Date.now() - new Date(date) < 10 * 60 * 1000, assert(
'run created date should be within the last 10 minutes'); Date.now() - new Date(date) < 10 * 60 * 1000,
'run created date should be within the last 10 minutes',
);
}); });
it('displays run inputs correctly', () => { it('displays run inputs correctly', async () => {
const paramValue = getValueFromDetailsTable('message'); const paramValue = await getValueFromDetailsTable('message');
assert.equal(paramValue, outputParameterValue, 'run message is not shown correctly'); assert.equal(paramValue, outputParameterValue, 'run message is not shown correctly');
}); });
it('switches back to graph tab', () => { it('switches back to graph tab', async () => {
$('button=Graph').click(); await $('button=Graph').click();
}); });
it('has a 4-node graph', () => { it('has a 4-node graph', async () => {
const nodeSelector = '.graphNode'; const nodeSelector = '.graphNode';
const nodes = $$(nodeSelector).length; const nodes = await $$(nodeSelector).length;
assert(nodes === 4, 'should have a 4-node graph, instead has: ' + nodes); assert(nodes === 4, 'should have a 4-node graph, instead has: ' + nodes);
}); });
it('opens the side panel when graph node is clicked', () => { it('opens the side panel when graph node is clicked', async () => {
$('.graphNode').click(); await $('.graphNode').click();
browser.pause(1000); await browser.pause(1000);
$('button=Logs').waitForVisible(); await $('button=Logs').waitForDisplayed();
}); });
it('shows logs from node', () => { it('shows logs from node', async () => {
$('button=Logs').click(); await $('button=Logs').click();
$('#logViewer').waitForVisible(); await $('#logViewer').waitForDisplayed();
browser.waitUntil(() => { await browser.waitUntil(async () => {
const logs = $('#logViewer').getText(); const logs = await $('#logViewer').getText();
return logs.indexOf(outputParameterValue + ' from node: ') > -1; return logs.indexOf(outputParameterValue + ' from node: ') > -1;
}, waitTimeout); }, waitTimeout);
}); });
it('navigates to the runs page', () => { it('navigates to the runs page', async () => {
$('#runsBtn').click(); await $('#runsBtn').click();
browser.waitUntil(() => { await browser.waitUntil(async () => {
return new URL(browser.getUrl()).hash.startsWith('#/runs'); return new URL(await browser.getUrl()).hash.startsWith('#/runs');
}, waitTimeout); }, waitTimeout);
}); });
it('creates a new run without selecting an experiment', () => { it('creates a new run without selecting an experiment', async () => {
$('#createNewRunBtn').waitForVisible(); await $('#createNewRunBtn').waitForDisplayed();
$('#createNewRunBtn').click(); await $('#createNewRunBtn').click();
$('#choosePipelineBtn').waitForVisible(); await $('#choosePipelineBtn').waitForDisplayed();
$('#choosePipelineBtn').click(); await $('#choosePipelineBtn').click();
$('.tableRow').waitForVisible(); await $('.tableRow').waitForDisplayed();
$('.tableRow').click(); await $('.tableRow').click();
$('#usePipelineBtn').click(); await $('#usePipelineBtn').click();
$('#pipelineSelectorDialog').waitForVisible(waitTimeout, true); await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: waitTimeout, reverse: true });
browser.keys('Tab'); await browser.keys('Tab');
browser.keys(runWithoutExperimentName); await browser.keys(runWithoutExperimentName);
browser.keys('Tab'); await browser.keys('Tab');
browser.keys(runWithoutExperimentDescription); await browser.keys(runWithoutExperimentDescription);
// Skip over "choose experiment" button // Skip over "choose experiment" button
browser.keys('Tab'); await browser.keys('Tab');
// Skip over service account help button // Skip over service account help button
browser.keys('Tab'); await browser.keys('Tab');
// Skip over "service account" textbox // Skip over "service account" textbox
browser.keys('Tab'); await browser.keys('Tab');
// Skip over "Run Type" radio button // Skip over "Run Type" radio button
browser.keys('Tab'); await browser.keys('Tab');
browser.keys('Tab'); await browser.keys('Tab');
browser.keys(outputParameterValue); await browser.keys(outputParameterValue);
// Deploy // Deploy
$('#startNewRunBtn').click(); await $('#startNewRunBtn').click();
}); });
it('redirects back to all runs page', () => { it('redirects back to all runs page', async () => {
browser.waitUntil(() => { await browser.waitUntil(
return (new URL(browser.getUrl())).hash === '#/runs'; async () => {
}, waitTimeout, `URL was: ${new URL(browser.getUrl())}`); return new URL(await browser.getUrl()).hash === '#/runs';
},
waitTimeout,
`URL was: ${new URL(await browser.getUrl())}`,
);
}); });
it('displays both runs in all runs page', () => { it('displays both runs in all runs page', async () => {
$('.tableRow').waitForVisible(); await $('.tableRow').waitForDisplayed();
const rows = $$('.tableRow').length; const rows = await $$('.tableRow').length;
assert(rows === 2, 'there should now be two runs in the table, instead there are: ' + rows); assert(rows === 2, 'there should now be two runs in the table, instead there are: ' + rows);
}); });
it('navigates back to the experiment list', () => { it('navigates back to the experiment list', async () => {
$('button=Experiments').click(); await $('button=Experiments').click();
browser.waitUntil(() => { await browser.waitUntil(async () => {
return new URL(browser.getUrl()).hash.startsWith('#/experiments'); return new URL(await browser.getUrl()).hash.startsWith('#/experiments');
}, waitTimeout); }, waitTimeout);
}); });
it('displays both experiments in the list', () => { it('displays both experiments in the list', async () => {
$('.tableRow').waitForVisible(); await $('.tableRow').waitForDisplayed();
const rows = $$('.tableRow').length; const rows = await $$('.tableRow').length;
assert(rows === 2, 'there should now be two experiments in the table, instead there are: ' + rows); assert(
rows === 2,
'there should now be two experiments in the table, instead there are: ' + rows,
);
}); });
it('filters the experiment list', () => { it('filters the experiment list', async () => {
// Enter "hello" into filter bar // Enter "hello" into filter bar
browser.click('#tableFilterBox'); await $('#tableFilterBox').click();
browser.keys(experimentName.substring(0, 5)); await browser.keys(experimentName.substring(0, 5));
// Wait for the list to refresh // Wait for the list to refresh
browser.pause(2000); await browser.pause(2000);
$('.tableRow').waitForVisible(); await $('.tableRow').waitForDisplayed();
const rows = $$('.tableRow').length; const rows = await $$('.tableRow').length;
assert(rows === 1, 'there should now be one experiment in the table, instead there are: ' + rows); assert(
rows === 1,
'there should now be one experiment in the table, instead there are: ' + rows,
);
}); });
//TODO: enable this after we change the pipeline to a unique name such that deleting this //TODO: enable this after we change the pipeline to a unique name such that deleting this
// pipeline will not jeopardize the concurrent basic e2e tests. // pipeline will not jeopardize the concurrent basic e2e tests.
// it('deletes the uploaded pipeline', () => { // it('deletes the uploaded pipeline', async () => {
// $('#pipelinesBtn').click(); // await $('#pipelinesBtn').click();
// //
// browser.waitForVisible('.tableRow', waitTimeout); // await $('.tableRow').waitForDisplayed({timeout: waitTimeout});
// $('.tableRow').click(); // await $('.tableRow').click();
// $('#deleteBtn').click(); // await $('#deleteBtn').click();
// $('.dialogButton').click(); // await $('.dialogButton').click();
// $('.dialog').waitForVisible(waitTimeout, true); // await $('.dialog').waitForDisplayed({timeout: waitTimeout, reverse:true});
// }); // });
}); });

6481
test/frontend-integration-test/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

16
test/frontend-integration-test/package.json
View File

@ -3,13 +3,15 @@
"version": "0.0.1", "version": "0.0.1",
"description": "Visual regression end to end test package for ML Pipelines", "description": "Visual regression end to end test package for ML Pipelines",
"dependencies": { "dependencies": {
"lodash": ">=4.17.21", "@wdio/cli": "^8.3.2",
"mocha": "^5.2.0", "@wdio/junit-reporter": "^8.3.0",
"wait-port": "^0.2.2", "@wdio/local-runner": "^8.3.3",
"wdio-junit-reporter": "^0.4.4", "@wdio/mocha-framework": "^8.3.0",
"wdio-mocha-framework": "^0.6.2", "@wdio/selenium-standalone-service": "^8.3.2",
"wdio-selenium-standalone-service": "0.0.10", "@wdio/spec-reporter": "^8.3.0",
"webdriverio": "^4.14.1" "mocha": "^10.2.0",
"wait-port": "^1.0.4",
"webdriverio": "^8.3.2"
}, },
"scripts": { "scripts": {
"docker": "docker build -t gcr.io/ml-pipeline/e2e-tests .", "docker": "docker build -t gcr.io/ml-pipeline/e2e-tests .",

13
test/frontend-integration-test/selenium-standalone-chrome-gcloud-nodejs.Docker/Dockerfile
View File

@ -1,8 +1,8 @@
FROM selenium/standalone-chrome:3.141.59-oxygen FROM selenium/standalone-chrome:109.0.5414.119-chromedriver-109.0.5414.74-20230131
USER root USER root
ARG CLOUD_SDK_VERSION=279.0.0 ARG CLOUD_SDK_VERSION=416.0.0
ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION
ARG INSTALL_COMPONENTS=kubectl ARG INSTALL_COMPONENTS=kubectl
@ -10,8 +10,8 @@ ENV PATH "$PATH:/opt/google-cloud-sdk/bin/"
RUN apt-get update -qqy && apt-get install -qqy \ RUN apt-get update -qqy && apt-get install -qqy \
curl \ curl \
gcc \ gcc \
python-dev \ python3-dev \
python-pip \ python3-pip \
apt-transport-https \ apt-transport-https \
lsb-release \ lsb-release \
openssh-client \ openssh-client \
@ -19,14 +19,13 @@ RUN apt-get update -qqy && apt-get install -qqy \
gnupg \ gnupg \
&& \ && \
pip install -U crcmod && \ pip install -U crcmod && \
export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \
echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
apt-get update && apt-get install -y google-cloud-sdk=${CLOUD_SDK_VERSION}-0 $INSTALL_COMPONENTS && \ apt-get update && apt-get install -y google-cloud-sdk=${CLOUD_SDK_VERSION}-0 $INSTALL_COMPONENTS && \
rm -rf /var/lib/apt/lists/* /tmp/* /usr/share/locale/* /usr/share/i18n/locales/* rm -rf /var/lib/apt/lists/* /tmp/* /usr/share/locale/* /usr/share/i18n/locales/*
RUN curl --silent --show-error --location https://deb.nodesource.com/setup_8.x | bash - && \ RUN curl --silent --show-error --location https://deb.nodesource.com/setup_14.x | bash - && \
apt-get install -y nodejs apt-get install -y nodejs
USER seluser USER seluser

52
test/frontend-integration-test/wdio.conf.js
View File

@ -15,41 +15,45 @@
const debug = process.env.DEBUG == '1' || process.env.DEBUG == 'true'; const debug = process.env.DEBUG == '1' || process.env.DEBUG == 'true';
exports.config = { exports.config = {
runner: 'local',
host: '127.0.0.1',
port: 4444,
maxInstances: 1, maxInstances: 1,
baseUrl: 'http://localhost:3000', baseUrl: 'http://localhost:3000',
capabilities: [{ capabilities: [
maxInstances: 1, {
browserName: 'chrome', maxInstances: 1,
chromeOptions: { browserName: 'chrome',
args: debug ? [] : ['--headless', '--disable-gpu'], 'goog:chromeOptions': {
args: ['--headless', '--disable-gpu'],
},
}, },
}], ],
coloredLogs: true, coloredLogs: true,
connectionRetryCount: 3, connectionRetryCount: 3,
connectionRetryTimeout: 90000, connectionRetryTimeout: 90000,
deprecationWarnings: false, deprecationWarnings: false,
framework: 'mocha', framework: 'mocha',
host: '127.0.0.1',
port: 4444,
mochaOpts: { mochaOpts: {
ui: 'bdd',
// units: ms // units: ms
//TODO:Reduce the timeout once the tests become shorter timeout: 1200000,
timeout: debug ? 99999999 : 1200000,
}, },
logLevel: 'silent', logLevel: debug ? 'info' : 'silent',
reporters: debug ? [] : ['dot', 'junit'], reporters: [
reporterOptions: debug ? {} : { 'spec',
junit: { [
outputDir: './', 'junit',
outputFileFormat: { {
single: () => 'junit_FrontendIntegrationTestOutput.xml', outputDir: './',
} outputFileFormat: function (options) {
}, return 'junit_FrontendIntegrationTestOutput.xml';
}, },
services: debug ? ['selenium-standalone'] : [], },
specs: [ ],
'./helloworld.spec.js',
], ],
services: debug ? [['selenium-standalone', { drivers: { chrome: 'latest' } }]] : [],
specs: ['./helloworld.spec.js'],
sync: true, sync: true,
waitforTimeout: 10000, waitforTimeout: 10000,
} };