kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Integrate pipeline details from cloned object with v2 API.

This commit is contained in:
Joe Li 2023-03-16 17:09:03 +00:00
parent 90bb6515e6
commit a4f2391302
2 changed files with 69 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
frontend/src/pages/PipelineDetails.test.tsx
View File

@ -15,12 +15,15 @@
*/
import { render, waitFor } from '@testing-library/react';
import fs from 'fs';
import { graphlib } from 'dagre';
import { ReactWrapper, shallow, ShallowWrapper } from 'enzyme';
import * as React from 'react';
import { ApiExperiment } from '../apis/experiment';
import { ApiPipeline, ApiPipelineVersion } from '../apis/pipeline';
import { ApiResourceType, ApiRunDetail } from '../apis/run';
import { V2beta1Run } from 'src/apisv2beta1/run';
import { V2beta1RecurringRun } from 'src/apisv2beta1/recurringrun';
import { QUERY_PARAMS, RoutePage, RouteParams } from '../components/Router';
import { Apis } from '../lib/Apis';
import { ButtonKeys } from '../lib/Buttons';
@ -40,8 +43,8 @@ describe('PipelineDetails', () => {
const getPipelineSpy = jest.spyOn(Apis.pipelineServiceApi, 'getPipeline');
const getPipelineVersionSpy = jest.spyOn(Apis.pipelineServiceApi, 'getPipelineVersion');
const listPipelineVersionsSpy = jest.spyOn(Apis.pipelineServiceApi, 'listPipelineVersions');
const getRunSpy = jest.spyOn(Apis.runServiceApi, 'getRun');
const getRecurringRunSpy = jest.spyOn(Apis.jobServiceApi, 'getJob');
const getRunSpy = jest.spyOn(Apis.runServiceApiV2, 'getRun');
const getRecurringRunSpy = jest.spyOn(Apis.recurringRunServiceApi, 'getRecurringRun');
const getExperimentSpy = jest.spyOn(Apis.experimentServiceApi, 'getExperiment');
const deletePipelineVersionSpy = jest.spyOn(Apis.pipelineServiceApi, 'deletePipelineVersion');
const getPipelineVersionTemplateSpy = jest.spyOn(
@ -49,12 +52,15 @@ describe('PipelineDetails', () => {
'getPipelineVersionTemplate',
);
const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph');
const V2_PIPELINESPEC_PATH = 'src/data/test/xgboost_sample_pipeline.yaml';
const v2YamlTemplateString = fs.readFileSync(V2_PIPELINESPEC_PATH, 'utf8');
const v2PipelineSpec = WorkflowUtils.convertYamlToV2PipelineSpec(v2YamlTemplateString);
let tree: ShallowWrapper | ReactWrapper;
let testPipeline: ApiPipeline = {};
let testPipelineVersion: ApiPipelineVersion = {};
let testRun: ApiRunDetail = {};
let testRecurringRun: ApiJob = {};
let testRun: V2beta1Run = {};
let testRecurringRun: V2beta1RecurringRun = {};
function generateProps(fromRunSpec = false, fromRecurringRunSpec = false): PageProps {
let params = {};
@ -118,21 +124,17 @@ describe('PipelineDetails', () => {
};
testRun = {
run: {
id: 'test-run-id',
name: 'test run',
pipeline_spec: {
pipeline_id: 'run-pipeline-id',
},
},
display_name: 'test run',
experiment_id: '',
run_id: 'test-run-id',
pipeline_spec: {},
};
testRecurringRun = {
id: 'test-recurring-run-id',
name: 'test recurring run',
pipeline_spec: {
pipeline_id: 'run-pipeline-id',
},
display_name: 'test recurring run',
experiment_id:'',
recurring_run_id: 'test-recurring-run-id',
pipeline_spec: {},
};
getPipelineSpy.mockImplementation(() => Promise.resolve(testPipeline));
@ -184,8 +186,8 @@ describe('PipelineDetails', () => {
breadcrumbs: [
{ displayName: 'All runs', href: RoutePage.RUNS },
{
displayName: testRun.run!.name,
href: RoutePage.RUN_DETAILS.replace(':' + RouteParams.runId, testRun.run!.id!),
displayName: testRun.display_name,
href: RoutePage.RUN_DETAILS.replace(':' + RouteParams.runId, testRun.run_id!),
},
],
pageTitle: 'Pipeline details',
@ -207,10 +209,10 @@ describe('PipelineDetails', () => {
breadcrumbs: [
{ displayName: 'All recurring runs', href: RoutePage.RECURRING_RUNS },
{
displayName: testRecurringRun.name,
displayName: testRecurringRun.display_name,
href: RoutePage.RECURRING_RUN_DETAILS.replace(
':' + RouteParams.recurringRunId,
testRecurringRun.id!,
testRecurringRun.recurring_run_id!,
),
},
],
@ -224,9 +226,7 @@ describe('PipelineDetails', () => {
'shows all runs breadcrumbs, and "Pipeline details" as page title when the pipeline ' +
'comes from a run spec that has an experiment',
async () => {
testRun.run!.resource_references = [
{ key: { id: 'test-experiment-id', type: ApiResourceType.EXPERIMENT } },
];
testRun.experiment_id = 'test-experiment-id';
tree = shallow(<PipelineDetails {...generateProps(true)} />);
await getRunSpy;
await getExperimentSpy;
@ -244,8 +244,8 @@ describe('PipelineDetails', () => {
),
},
{
displayName: testRun.run!.name,
href: RoutePage.RUN_DETAILS.replace(':' + RouteParams.runId, testRun.run!.id!),
displayName: testRun.display_name,
href: RoutePage.RUN_DETAILS.replace(':' + RouteParams.runId, testRun.run_id!),
},
],
pageTitle: 'Pipeline details',
@ -258,9 +258,7 @@ describe('PipelineDetails', () => {
'shows all runs breadcrumbs, and "Pipeline details" as page title when the pipeline ' +
'comes from a recurring run spec that has an experiment',
async () => {
testRecurringRun.resource_references = [
{ key: { id: 'test-experiment-id', type: ApiResourceType.EXPERIMENT } },
];
testRecurringRun.experiment_id = 'test-experiment-id';
tree = shallow(<PipelineDetails {...generateProps(false, true)} />);
await getRecurringRunSpy;
await getExperimentSpy;
@ -278,10 +276,10 @@ describe('PipelineDetails', () => {
),
},
{
displayName: testRecurringRun.name,
displayName: testRecurringRun.display_name,
href: RoutePage.RECURRING_RUN_DETAILS.replace(
':' + RouteParams.recurringRunId,
testRecurringRun.id!,
testRecurringRun.recurring_run_id!,
),
},
],
@ -292,10 +290,7 @@ describe('PipelineDetails', () => {
);
it('parses the workflow source in embedded pipeline spec as JSON and then converts it to YAML (v1)', async () => {
testRun.run!.pipeline_spec = {
pipeline_id: 'run-pipeline-id',
workflow_manifest: '{"spec": {"arguments": {"parameters": [{"name": "output"}]}}}',
};
testRun.pipeline_spec = '{"spec": {"arguments": {"parameters": [{"name": "output"}]}}}';
tree = shallow(<PipelineDetails {...generateProps(true)} />);
await TestUtils.flushPromises();
@ -306,11 +301,7 @@ describe('PipelineDetails', () => {
});
it('directly uses pipeline manifest as template string (v2)', async () => {
testRun.run!.pipeline_spec = {
pipeline_id: 'run-pipeline-id',
workflow_manifest: '{"spec": {"arguments": {"parameters": [{"name": "output"}]}}}',
pipeline_manifest: 'spec:\n arguments:\n parameters:\n - name: output\n',
};
testRun.pipeline_spec = 'spec:\n arguments:\n parameters:\n - name: output\n';
tree = shallow(<PipelineDetails {...generateProps(true)} />);
await TestUtils.flushPromises();
@ -321,11 +312,7 @@ describe('PipelineDetails', () => {
});
it('directly uses pipeline manifest from recurring run as template string (v2)', async () => {
testRecurringRun.pipeline_spec = {
pipeline_id: 'run-pipeline-id',
workflow_manifest: '{"spec": {"arguments": {"parameters": [{"name": "output"}]}}}',
pipeline_manifest: 'spec:\n arguments:\n parameters:\n - name: output\n',
};
testRecurringRun.pipeline_spec = 'spec:\n arguments:\n parameters:\n - name: output\n';
tree = shallow(<PipelineDetails {...generateProps(false, true)} />);
await TestUtils.flushPromises();
@ -336,10 +323,7 @@ describe('PipelineDetails', () => {
});
it('shows load error banner when failing to parse the workflow source in embedded pipeline spec', async () => {
testRun.run!.pipeline_spec = {
pipeline_id: 'run-pipeline-id',
workflow_manifest: 'not valid JSON',
};
testRun.pipeline_spec = 'not valid JSON';
render(<PipelineDetails {...generateProps(true)} />);
await waitFor(() => {
@ -348,7 +332,7 @@ describe('PipelineDetails', () => {
expect.objectContaining({
additionalInfo: 'Unexpected token o in JSON at position 1',
message: `Failed to parse pipeline spec from run with ID: ${
testRun.run!.id
testRun.run_id
}. Click Details for more information.`,
mode: 'error',
}),
@ -372,9 +356,7 @@ describe('PipelineDetails', () => {
});
it('shows load error banner when failing to get experiment details, when loading from run spec', async () => {
testRun.run!.resource_references = [
{ key: { id: 'test-experiment-id', type: ApiResourceType.EXPERIMENT } },
];
testRun.experiment_id = 'test-experiment-id';
TestUtils.makeErrorResponse(getExperimentSpy, 'woops');
tree = shallow(<PipelineDetails {...generateProps(true)} />);
await getPipelineSpy;
@ -506,7 +488,7 @@ describe('PipelineDetails', () => {
newRunBtn!.action();
expect(historyPushSpy).toHaveBeenCalledTimes(1);
expect(historyPushSpy).toHaveBeenLastCalledWith(
RoutePage.NEW_RUN + `?${QUERY_PARAMS.fromRunId}=${testRun.run!.id}`,
RoutePage.NEW_RUN + `?${QUERY_PARAMS.fromRunId}=${testRun.run_id}`,
);
});

56
frontend/src/pages/PipelineDetails.tsx
View File

@ -48,6 +48,8 @@ import PipelineDetailsV1 from './PipelineDetailsV1';
import PipelineDetailsV2 from './PipelineDetailsV2';
import { ApiRunDetail } from 'src/apis/run';
import { ApiJob } from 'src/apis/job';
import { V2beta1Run } from 'src/apisv2beta1/run';
import { V2beta1RecurringRun } from 'src/apisv2beta1/recurringrun';
interface PipelineDetailsState {
graph: dagre.graphlib.Graph | null;
@ -65,8 +67,8 @@ type Origin = {
isRecurring: boolean;
runId: string | null;
recurringRunId: string | null;
run?: ApiRunDetail;
recurringRun?: ApiJob;
run?: V2beta1Run;
recurringRun?: V2beta1RecurringRun;
};
class PipelineDetails extends Page<{}, PipelineDetailsState> {
@ -236,24 +238,36 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> {
const msgRunOrRecurringRun = origin.isRecurring ? 'recurring run' : 'run';
try {
if (origin.isRecurring) {
origin.recurringRun = await Apis.jobServiceApi.getJob(origin.recurringRunId!);
origin.recurringRun = await Apis.recurringRunServiceApi.getRecurringRun(origin.recurringRunId!);
} else {
origin.run = await Apis.runServiceApi.getRun(origin.runId!);
origin.run = await Apis.runServiceApiV2.getRun(origin.runId!);
}
const pipelineManifest = origin.isRecurring
? origin.recurringRun!.pipeline_spec?.pipeline_manifest
: origin.run!.run?.pipeline_spec?.pipeline_manifest;
const pipelineVersionIdFromOrigin = origin.isRecurring ? origin.recurringRun?.pipeline_version_id : origin.run?.pipeline_version_id;
let templateStrFromOrigin = '';
if (pipelineVersionIdFromOrigin) {
version = await Apis.pipelineServiceApi.getPipelineVersion(pipelineVersionIdFromOrigin);
const response = await Apis.pipelineServiceApi.getPipelineVersionTemplate(pipelineVersionIdFromOrigin);
templateStrFromOrigin = response.template || '';
}
let pipelineManifest = '';
if (origin.run?.pipeline_spec) {
pipelineManifest = JsYaml.safeDump(origin.run.pipeline_spec);
}
if (origin.recurringRun?.pipeline_spec) {
pipelineManifest = JsYaml.safeDump(origin.recurringRun.pipeline_spec);
}
// V1: Convert the run's pipeline spec to YAML to be displayed as the pipeline's source.
// V2: Use the pipeline spec string directly because it can be translated in JSON format.
if (isFeatureEnabled(FeatureKey.V2_ALPHA) && pipelineManifest) {
templateString = pipelineManifest;
// V2: Use the pipeline spec string or original template string directly
// because it can be translated in JSON format.
if (isFeatureEnabled(FeatureKey.V2_ALPHA) && (templateStrFromOrigin || pipelineManifest)) {
templateString = pipelineManifest ? pipelineManifest : templateStrFromOrigin;
} else {
try {
const workflowManifestString =
RunUtils.getWorkflowManifest(
origin.isRecurring ? origin.recurringRun : origin.run!.run,
) || '';
const workflowManifestString = pipelineManifest || '';
const workflowManifest = JSON.parse(workflowManifestString || '{}');
try {
templateString = WorkflowUtils.isPipelineSpec(workflowManifestString)
@ -262,13 +276,13 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> {
} catch (err) {
await this.showPageError(
`Failed to parse pipeline spec from ${msgRunOrRecurringRun} with ID: ${
origin.isRecurring ? origin.recurringRun!.id : origin.run!.run!.id
origin.isRecurring ? origin.recurringRun!.recurring_run_id : origin.run!.run_id
}.`,
err,
);
logger.error(
`Failed to convert pipeline spec YAML from ${msgRunOrRecurringRun} with ID: ${
origin.isRecurring ? origin.recurringRun!.id : origin.run!.run!.id
origin.isRecurring ? origin.recurringRun!.recurring_run_id : origin.run!.run_id
}.`,
err,
);
@ -276,22 +290,20 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> {
} catch (err) {
await this.showPageError(
`Failed to parse pipeline spec from ${msgRunOrRecurringRun} with ID: ${
origin.isRecurring ? origin.recurringRun!.id : origin.run!.run!.id
origin.isRecurring ? origin.recurringRun!.recurring_run_id : origin.run!.run_id
}.`,
err,
);
logger.error(
`Failed to parse pipeline spec JSON from ${msgRunOrRecurringRun} with ID: ${
origin.isRecurring ? origin.recurringRun!.id : origin.run!.run!.id
origin.isRecurring ? origin.recurringRun!.recurring_run_id : origin.run!.run_id
}.`,
err,
);
}
}
const relatedExperimentId = RunUtils.getFirstExperimentReferenceId(
origin.isRecurring ? origin.recurringRun : origin.run!.run,
);
const relatedExperimentId = origin.isRecurring ? origin.recurringRun?.experiment_id : origin.run?.experiment_id;
let experiment: ApiExperiment | undefined;
if (relatedExperimentId) {
experiment = await Apis.experimentServiceApi.getExperiment(relatedExperimentId);
@ -316,7 +328,7 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> {
});
}
breadcrumbs.push({
displayName: origin.isRecurring ? origin.recurringRun!.name! : origin.run!.run!.name!,
displayName: origin.isRecurring ? origin.recurringRun!.display_name! : origin.run!.display_name!,
href: origin.isRecurring
? RoutePage.RECURRING_RUN_DETAILS.replace(
':' + RouteParams.recurringRunId,