kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(frontend): UX change to support downloading directory artifacts. Fixes #3667 (#4696) 

* feat(frontend): change artifact link to download without extracting. Fixes #3667

* update artifact preview UX

* Fix tests

* add frontend unit tests

* fix

* move artifact integration tests to its own file

* fixed flaky aws-helper test

* refactor and add ui server integration tests

* Update UX according to feedback

* update snapshots

* Update minio-helper.ts

* update
This commit is contained in:
Yuan (Bob) Gong 2020-11-03 21:36:15 +08:00 committed by GitHub
parent 38946d88f8
commit e3992faf83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 890 additions and 628 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

497
frontend/server/app.test.ts
View File

@ -11,29 +11,21 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { PassThrough } from 'stream';
import express from 'express';
import fetch from 'node-fetch';
import requests from 'supertest';
import { Client as MinioClient } from 'minio';
import { Storage as GCSStorage } from '@google-cloud/storage';
import { UIServer } from './app';
import { loadConfigs } from './configs';
import * as minioHelper from './minio-helper';
import { TEST_ONLY as K8S_TEST_EXPORT } from './k8s-helper';
import * as serverInfo from './helpers/server-info';
import { Server } from 'http';
import { commonSetup } from './integration-tests/test-helper';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
jest.mock('minio');
jest.mock('node-fetch');
jest.mock('@google-cloud/storage');
jest.mock('./minio-helper');
// TODO: move sections of tests here to individual files in `frontend/server/integration-tests/`
// for better organization and shorter/more focused tests.
@ -46,13 +38,6 @@ describe('UIServer apis', () => {
const commitHash = 'abcdefg';
const { argv, buildDate, indexHtmlContent } = commonSetup({ tagName, commitHash });
beforeEach(() => {
const consoleInfoSpy = jest.spyOn(global.console, 'info');
consoleInfoSpy.mockImplementation(() => null);
const consoleLogSpy = jest.spyOn(global.console, 'log');
consoleLogSpy.mockImplementation(() => null);
});
afterEach(() => {
if (app) {
app.close();
@ -162,488 +147,6 @@ describe('UIServer apis', () => {
});
});
describe('/artifacts/get', () => {
it('responds with a minio artifact if source=minio', done => {
const artifactContent = 'hello world';
const mockedMinioClient: jest.Mock = MinioClient as any;
const mockedGetObjectStream: jest.Mock = minioHelper.getObjectStream as any;
const objStream = new PassThrough();
objStream.end(artifactContent);
mockedGetObjectStream.mockImplementationOnce(opt =>
opt.bucket === 'ml-pipeline' && opt.key === 'hello/world.txt'
? Promise.resolve(objStream)
: Promise.reject('Unable to retrieve minio artifact.'),
);
const configs = loadConfigs(argv, {
MINIO_ACCESS_KEY: 'minio',
MINIO_HOST: 'minio-service',
MINIO_NAMESPACE: 'kubeflow',
MINIO_PORT: '9000',
MINIO_SECRET_KEY: 'minio123',
MINIO_SSL: 'false',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=minio&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedMinioClient).toBeCalledWith({
accessKey: 'minio',
endPoint: 'minio-service.kubeflow',
port: 9000,
secretKey: 'minio123',
useSSL: false,
});
done(err);
});
});
it('responds with a s3 artifact if source=s3', done => {
const artifactContent = 'hello world';
const mockedMinioClient: jest.Mock = minioHelper.createMinioClient as any;
const mockedGetObjectStream: jest.Mock = minioHelper.getObjectStream as any;
const stream = new PassThrough();
stream.write(artifactContent);
stream.end();
mockedGetObjectStream.mockImplementationOnce(opt =>
opt.bucket === 'ml-pipeline' && opt.key === 'hello/world.txt'
? Promise.resolve(stream)
: Promise.reject('Unable to retrieve s3 artifact.'),
);
const configs = loadConfigs(argv, {
AWS_ACCESS_KEY_ID: 'aws123',
AWS_SECRET_ACCESS_KEY: 'awsSecret123',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedMinioClient).toBeCalledWith({
accessKey: 'aws123',
endPoint: 's3.amazonaws.com',
secretKey: 'awsSecret123',
});
done(err);
});
});
it('responds with partial s3 artifact if peek=5 flag is set', done => {
const artifactContent = 'hello world';
const mockedMinioClient: jest.Mock = minioHelper.createMinioClient as any;
const mockedGetObjectStream: jest.Mock = minioHelper.getObjectStream as any;
const stream = new PassThrough();
stream.write(artifactContent);
stream.end();
mockedGetObjectStream.mockImplementationOnce(opt =>
opt.bucket === 'ml-pipeline' && opt.key === 'hello/world.txt'
? Promise.resolve(stream)
: Promise.reject('Unable to retrieve s3 artifact.'),
);
const configs = loadConfigs(argv, {
AWS_ACCESS_KEY_ID: 'aws123',
AWS_SECRET_ACCESS_KEY: 'awsSecret123',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
.expect(200, artifactContent.slice(0, 5), err => {
expect(mockedMinioClient).toBeCalledWith({
accessKey: 'aws123',
endPoint: 's3.amazonaws.com',
secretKey: 'awsSecret123',
});
done(err);
});
});
it('responds with a http artifact if source=http', done => {
const artifactContent = 'hello world';
mockedFetch.mockImplementationOnce((url: string, opts: any) =>
url === 'http://foo.bar/ml-pipeline/hello/world.txt'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=http&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedFetch).toBeCalledWith('http://foo.bar/ml-pipeline/hello/world.txt', {
headers: {},
});
done(err);
});
});
it('responds with partial http artifact if peek=5 flag is set', done => {
const artifactContent = 'hello world';
const mockedFetch: jest.Mock = fetch as any;
mockedFetch.mockImplementationOnce((url: string, opts: any) =>
url === 'http://foo.bar/ml-pipeline/hello/world.txt'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=http&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
.expect(200, artifactContent.slice(0, 5), err => {
expect(mockedFetch).toBeCalledWith('http://foo.bar/ml-pipeline/hello/world.txt', {
headers: {},
});
done(err);
});
});
it('responds with a https artifact if source=https', done => {
const artifactContent = 'hello world';
mockedFetch.mockImplementationOnce((url: string, opts: any) =>
url === 'https://foo.bar/ml-pipeline/hello/world.txt' &&
opts.headers.Authorization === 'someToken'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_AUTHORIZATION_DEFAULT_VALUE: 'someToken',
HTTP_AUTHORIZATION_KEY: 'Authorization',
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=https&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedFetch).toBeCalledWith('https://foo.bar/ml-pipeline/hello/world.txt', {
headers: {
Authorization: 'someToken',
},
});
done(err);
});
});
it('responds with a https artifact using the inherited header if source=https and http authorization key is provided.', done => {
const artifactContent = 'hello world';
mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
url === 'https://foo.bar/ml-pipeline/hello/world.txt'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_AUTHORIZATION_KEY: 'Authorization',
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=https&bucket=ml-pipeline&key=hello%2Fworld.txt')
.set('Authorization', 'inheritedToken')
.expect(200, artifactContent, err => {
expect(mockedFetch).toBeCalledWith('https://foo.bar/ml-pipeline/hello/world.txt', {
headers: {
Authorization: 'inheritedToken',
},
});
done(err);
});
});
it('responds with a gcs artifact if source=gcs', done => {
const artifactContent = 'hello world';
const mockedGcsStorage: jest.Mock = GCSStorage as any;
const stream = new PassThrough();
stream.write(artifactContent);
stream.end();
mockedGcsStorage.mockImplementationOnce(() => ({
bucket: () => ({
getFiles: () =>
Promise.resolve([[{ name: 'hello/world.txt', createReadStream: () => stream }]]),
}),
}));
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=gcs&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent + '\n', done);
});
it('responds with a partial gcs artifact if peek=5 is set', done => {
const artifactContent = 'hello world';
const mockedGcsStorage: jest.Mock = GCSStorage as any;
const stream = new PassThrough();
stream.end(artifactContent);
mockedGcsStorage.mockImplementationOnce(() => ({
bucket: () => ({
getFiles: () =>
Promise.resolve([[{ name: 'hello/world.txt', createReadStream: () => stream }]]),
}),
}));
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=gcs&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
.expect(200, artifactContent.slice(0, 5), done);
});
it('responds with a volume artifact if source=volume', done => {
const artifactContent = 'hello world';
const tempPath = path.join(fs.mkdtempSync(os.tmpdir()), 'content');
fs.writeFileSync(tempPath, artifactContent);
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: path.dirname(tempPath),
subPath: 'subartifact',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=volume&bucket=artifact&key=subartifact/content')
.expect(200, artifactContent, done);
});
it('responds with a partial volume artifact if peek=5 is set', done => {
const artifactContent = 'hello world';
const tempPath = path.join(fs.mkdtempSync(os.tmpdir()), 'content');
fs.writeFileSync(tempPath, artifactContent);
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: path.dirname(tempPath),
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=artifact&key=content&peek=5`)
.expect(200, artifactContent.slice(0, 5), done);
});
it('responds error with a not exist volume', done => {
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
metadata: {
name: 'ml-pipeline-ui',
},
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: '/foo/bar/path',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=notexist&key=content`)
.expect(
404,
'Failed to open volume://notexist/content, Cannot find file "volume://notexist/content" in pod "ml-pipeline-ui": volume "notexist" not configured',
done,
);
});
it('responds error with a not exist volume mount path if source=volume', done => {
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
metadata: {
name: 'ml-pipeline-ui',
},
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: '/foo/bar/path',
subPath: 'subartifact',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=artifact&key=notexist/config`)
.expect(
404,
'Failed to open volume://artifact/notexist/config, Cannot find file "volume://artifact/notexist/config" in pod "ml-pipeline-ui": volume "artifact" not mounted or volume "artifact" with subPath (which is prefix of notexist/config) not mounted',
done,
);
});
it('responds error with a not exist volume mount artifact if source=volume', done => {
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: '/foo/bar',
subPath: 'subartifact',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=artifact&key=subartifact/notxist.csv`)
.expect(
500,
"Failed to open volume://artifact/subartifact/notxist.csv: Error: ENOENT: no such file or directory, stat '/foo/bar/notxist.csv'",
done,
);
});
});
describe('/system', () => {
describe('/cluster-name', () => {
it('responds with cluster name data from gke metadata', done => {

28
frontend/server/app.ts
View File

@ -118,13 +118,37 @@ function createUIServer(options: UIConfigs) {
/** Artifact */
registerHandler(
app.get,
'/artifacts/get',
'/artifacts/*',
getArtifactsProxyHandler({
enabled: options.artifacts.proxy.enabled,
namespacedServiceGetter: getArtifactServiceGetter(options.artifacts.proxy),
}),
);
registerHandler(app.get, '/artifacts/get', getArtifactsHandler(options.artifacts));
// /artifacts/get endpoint tries to extract the artifact to return pure text content
registerHandler(
app.get,
'/artifacts/get',
getArtifactsHandler({
artifactsConfigs: options.artifacts,
useParameter: false,
tryExtract: true,
}),
);
// /artifacts/ endpoint downloads the artifact as is, it does not try to unzip or untar.
registerHandler(
app.get,
// The last * represents object key. Key could contain special characters like '/',
// so we cannot use `:key` as the placeholder.
// It is important to include the original object's key at the end of the url, because
// browser automatically determines file extension by the url. A wrong extension may affect
// whether the file can be opened by the correct application by default.
'/artifacts/:source/:bucket/*',
getArtifactsHandler({
artifactsConfigs: options.artifacts,
useParameter: true,
tryExtract: false,
}),
);
/** Authorize function */
const authorizeFn = getAuthorizeFn(options.auth, { apiServerAddress });

4
frontend/server/aws-helper.test.ts
View File

@ -88,7 +88,7 @@ describe('awsInstanceProfileCredentials', () => {
mockedFetch.mockImplementation(mockFetch);
expect(await awsInstanceProfileCredentials.ok()).toBeFalsy();
expect(awsInstanceProfileCredentials.getCredentials).not.toThrow();
expect(async () => await awsInstanceProfileCredentials.getCredentials()).not.toThrow();
expect(await awsInstanceProfileCredentials.getCredentials()).toBeUndefined();
});
@ -99,7 +99,7 @@ describe('awsInstanceProfileCredentials', () => {
mockedFetch.mockImplementation(mockFetch);
expect(await awsInstanceProfileCredentials.ok()).toBeFalsy();
expect(awsInstanceProfileCredentials.getCredentials).not.toThrow();
expect(async () => await awsInstanceProfileCredentials.getCredentials()).not.toThrow();
expect(await awsInstanceProfileCredentials.getCredentials()).toBeUndefined();
});
});

36
frontend/server/handlers/artifacts.ts
View File

@ -23,7 +23,6 @@ import proxy from 'http-proxy-middleware';
import { HACK_FIX_HPM_PARTIAL_RESPONSE_HEADERS } from '../consts';
import * as fs from 'fs';
import { V1Container } from '@kubernetes/client-node/dist/api';
/**
* ArtifactsQueryStrings describes the expected query strings key value pairs
@ -44,17 +43,29 @@ interface ArtifactsQueryStrings {
* Returns an artifact handler which retrieve an artifact from the corresponding
* backend (i.e. gcs, minio, s3, http/https).
* @param artifactsConfigs configs to retrieve the artifacts from the various backend.
* @param useParameter get bucket and key from parameter instead of query. When true, expect
* to be used in a route like `/artifacts/:source/:bucket/*`.
* @param tryExtract whether the handler try to extract content from *.tar.gz files.
*/
export function getArtifactsHandler(artifactsConfigs: {
aws: AWSConfigs;
http: HttpConfigs;
minio: MinioConfigs;
export function getArtifactsHandler({
artifactsConfigs,
useParameter,
tryExtract,
}: {
artifactsConfigs: {
aws: AWSConfigs;
http: HttpConfigs;
minio: MinioConfigs;
};
tryExtract: boolean;
useParameter: boolean;
}): Handler {
const { aws, http, minio } = artifactsConfigs;
return async (req, res) => {
const { source, bucket, key: encodedKey, peek = 0 } = req.query as Partial<
ArtifactsQueryStrings
>;
const source = useParameter ? req.params.source : req.query.source;
const bucket = useParameter ? req.params.bucket : req.query.bucket;
const key = useParameter ? req.params[0] : req.query.key;
const { peek = 0 } = req.query as Partial<ArtifactsQueryStrings>;
if (!source) {
res.status(500).send('Storage source is missing from artifact request');
return;
@ -63,11 +74,10 @@ export function getArtifactsHandler(artifactsConfigs: {
res.status(500).send('Storage bucket is missing from artifact request');
return;
}
if (!encodedKey) {
if (!key) {
res.status(500).send('Storage key is missing from artifact request');
return;
}
const key = decodeURIComponent(encodedKey);
console.log(`Getting storage artifact at: ${source}: ${bucket}/${key}`);
switch (source) {
case 'gcs':
@ -80,6 +90,7 @@ export function getArtifactsHandler(artifactsConfigs: {
bucket,
client: new MinioClient(minio),
key,
tryExtract,
},
peek,
)(req, res);
@ -161,7 +172,7 @@ function getHttpArtifactsHandler(
}
function getMinioArtifactHandler(
options: { bucket: string; key: string; client: MinioClient },
options: { bucket: string; key: string; client: MinioClient; tryExtract?: boolean },
peek: number = 0,
) {
return async (_: Request, res: Response) => {
@ -178,6 +189,7 @@ function getMinioArtifactHandler(
.pipe(new PreviewStream({ peek }))
.pipe(res);
} catch (err) {
console.error(err);
res
.status(500)
.send(`Failed to get object in bucket ${options.bucket} at path ${options.key}: ${err}`);
@ -360,7 +372,7 @@ export function getArtifactsProxyHandler({
}
return namespacedServiceGetter(namespace);
},
target: '/artifacts/get',
target: '/artifacts',
headers: HACK_FIX_HPM_PARTIAL_RESPONSE_HEADERS,
},
);

542
frontend/server/integration-tests/artifact-get.test.ts Normal file
View File

@ -0,0 +1,542 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { Storage as GCSStorage } from '@google-cloud/storage';
import * as fs from 'fs';
import * as minio from 'minio';
import fetch from 'node-fetch';
import * as os from 'os';
import * as path from 'path';
import { PassThrough } from 'stream';
import requests from 'supertest';
import { UIServer } from '../app';
import { loadConfigs } from '../configs';
import * as serverInfo from '../helpers/server-info';
import * as minioHelper from '../minio-helper';
import { commonSetup } from './test-helper';
const MinioClient = minio.Client;
jest.mock('minio');
jest.mock('node-fetch');
jest.mock('@google-cloud/storage');
const mockedFetch: jest.Mock = fetch as any;
describe('/artifacts', () => {
let app: UIServer;
const { argv } = commonSetup();
let artifactContent: any = 'hello world';
beforeEach(() => {
artifactContent = 'hello world'; // reset
const mockedMinioClient: jest.Mock = MinioClient as any;
mockedMinioClient.mockImplementation(function() {
return {
getObject: async (bucket: string, key: string) => {
const objStream = new PassThrough();
objStream.end(artifactContent);
if (bucket === 'ml-pipeline' && key === 'hello/world.txt') {
return objStream;
} else {
throw new Error(`Unable to retrieve ${bucket}/${key} artifact.`);
}
},
};
});
});
afterEach(() => {
if (app) {
app.close();
}
});
describe('/get', () => {
it('responds with a minio artifact if source=minio', done => {
const mockedMinioClient: jest.Mock = minio.Client as any;
const configs = loadConfigs(argv, {
MINIO_ACCESS_KEY: 'minio',
MINIO_HOST: 'minio-service',
MINIO_NAMESPACE: 'kubeflow',
MINIO_PORT: '9000',
MINIO_SECRET_KEY: 'minio123',
MINIO_SSL: 'false',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=minio&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedMinioClient).toBeCalledWith({
accessKey: 'minio',
endPoint: 'minio-service.kubeflow',
port: 9000,
secretKey: 'minio123',
useSSL: false,
});
done(err);
});
});
it('responds with a s3 artifact if source=s3', done => {
const mockedMinioClient: jest.Mock = jest.spyOn(minioHelper, 'createMinioClient') as any;
const configs = loadConfigs(argv, {
AWS_ACCESS_KEY_ID: 'aws123',
AWS_SECRET_ACCESS_KEY: 'awsSecret123',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedMinioClient).toBeCalledWith({
accessKey: 'aws123',
endPoint: 's3.amazonaws.com',
secretKey: 'awsSecret123',
});
done(err);
});
});
it('responds with partial s3 artifact if peek=5 flag is set', done => {
const mockedMinioClient = jest.spyOn(minioHelper, 'createMinioClient');
const configs = loadConfigs(argv, {
AWS_ACCESS_KEY_ID: 'aws123',
AWS_SECRET_ACCESS_KEY: 'awsSecret123',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
.expect(200, artifactContent.slice(0, 5), err => {
expect(mockedMinioClient).toBeCalledWith({
accessKey: 'aws123',
endPoint: 's3.amazonaws.com',
secretKey: 'awsSecret123',
});
done(err);
});
});
it('responds with a http artifact if source=http', done => {
const artifactContent = 'hello world';
mockedFetch.mockImplementationOnce((url: string, opts: any) =>
url === 'http://foo.bar/ml-pipeline/hello/world.txt'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=http&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedFetch).toBeCalledWith('http://foo.bar/ml-pipeline/hello/world.txt', {
headers: {},
});
done(err);
});
});
it('responds with partial http artifact if peek=5 flag is set', done => {
const artifactContent = 'hello world';
const mockedFetch: jest.Mock = fetch as any;
mockedFetch.mockImplementationOnce((url: string, opts: any) =>
url === 'http://foo.bar/ml-pipeline/hello/world.txt'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=http&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
.expect(200, artifactContent.slice(0, 5), err => {
expect(mockedFetch).toBeCalledWith('http://foo.bar/ml-pipeline/hello/world.txt', {
headers: {},
});
done(err);
});
});
it('responds with a https artifact if source=https', done => {
const artifactContent = 'hello world';
mockedFetch.mockImplementationOnce((url: string, opts: any) =>
url === 'https://foo.bar/ml-pipeline/hello/world.txt' &&
opts.headers.Authorization === 'someToken'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_AUTHORIZATION_DEFAULT_VALUE: 'someToken',
HTTP_AUTHORIZATION_KEY: 'Authorization',
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=https&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent, err => {
expect(mockedFetch).toBeCalledWith('https://foo.bar/ml-pipeline/hello/world.txt', {
headers: {
Authorization: 'someToken',
},
});
done(err);
});
});
it('responds with a https artifact using the inherited header if source=https and http authorization key is provided.', done => {
const artifactContent = 'hello world';
mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
url === 'https://foo.bar/ml-pipeline/hello/world.txt'
? Promise.resolve({
buffer: () => Promise.resolve(artifactContent),
body: new PassThrough().end(artifactContent),
})
: Promise.reject('Unable to retrieve http artifact.'),
);
const configs = loadConfigs(argv, {
HTTP_AUTHORIZATION_KEY: 'Authorization',
HTTP_BASE_URL: 'foo.bar/',
});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=https&bucket=ml-pipeline&key=hello%2Fworld.txt')
.set('Authorization', 'inheritedToken')
.expect(200, artifactContent, err => {
expect(mockedFetch).toBeCalledWith('https://foo.bar/ml-pipeline/hello/world.txt', {
headers: {
Authorization: 'inheritedToken',
},
});
done(err);
});
});
it('responds with a gcs artifact if source=gcs', done => {
const artifactContent = 'hello world';
const mockedGcsStorage: jest.Mock = GCSStorage as any;
const stream = new PassThrough();
stream.write(artifactContent);
stream.end();
mockedGcsStorage.mockImplementationOnce(() => ({
bucket: () => ({
getFiles: () =>
Promise.resolve([[{ name: 'hello/world.txt', createReadStream: () => stream }]]),
}),
}));
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=gcs&bucket=ml-pipeline&key=hello%2Fworld.txt')
.expect(200, artifactContent + '\n', done);
});
it('responds with a partial gcs artifact if peek=5 is set', done => {
const artifactContent = 'hello world';
const mockedGcsStorage: jest.Mock = GCSStorage as any;
const stream = new PassThrough();
stream.end(artifactContent);
mockedGcsStorage.mockImplementationOnce(() => ({
bucket: () => ({
getFiles: () =>
Promise.resolve([[{ name: 'hello/world.txt', createReadStream: () => stream }]]),
}),
}));
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=gcs&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
.expect(200, artifactContent.slice(0, 5), done);
});
it('responds with a volume artifact if source=volume', done => {
const artifactContent = 'hello world';
const tempPath = path.join(fs.mkdtempSync(os.tmpdir()), 'content');
fs.writeFileSync(tempPath, artifactContent);
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: path.dirname(tempPath),
subPath: 'subartifact',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/get?source=volume&bucket=artifact&key=subartifact/content')
.expect(200, artifactContent, done);
});
it('responds with a partial volume artifact if peek=5 is set', done => {
const artifactContent = 'hello world';
const tempPath = path.join(fs.mkdtempSync(os.tmpdir()), 'content');
fs.writeFileSync(tempPath, artifactContent);
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: path.dirname(tempPath),
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=artifact&key=content&peek=5`)
.expect(200, artifactContent.slice(0, 5), done);
});
it('responds error with a not exist volume', done => {
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
metadata: {
name: 'ml-pipeline-ui',
},
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: '/foo/bar/path',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=notexist&key=content`)
.expect(
404,
'Failed to open volume://notexist/content, Cannot find file "volume://notexist/content" in pod "ml-pipeline-ui": volume "notexist" not configured',
done,
);
});
it('responds error with a not exist volume mount path if source=volume', done => {
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
metadata: {
name: 'ml-pipeline-ui',
},
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: '/foo/bar/path',
subPath: 'subartifact',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=artifact&key=notexist/config`)
.expect(
404,
'Failed to open volume://artifact/notexist/config, Cannot find file "volume://artifact/notexist/config" in pod "ml-pipeline-ui": volume "artifact" not mounted or volume "artifact" with subPath (which is prefix of notexist/config) not mounted',
done,
);
});
it('responds error with a not exist volume mount artifact if source=volume', done => {
jest.spyOn(serverInfo, 'getHostPod').mockImplementation(() =>
Promise.resolve([
{
spec: {
containers: [
{
volumeMounts: [
{
name: 'artifact',
mountPath: '/foo/bar',
subPath: 'subartifact',
},
],
name: 'ml-pipeline-ui',
},
],
volumes: [
{
name: 'artifact',
persistentVolumeClaim: {
claimName: 'artifact_pvc',
},
},
],
},
} as any,
undefined,
]),
);
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get(`/artifacts/get?source=volume&bucket=artifact&key=subartifact/notxist.csv`)
.expect(
500,
"Failed to open volume://artifact/subartifact/notxist.csv: Error: ENOENT: no such file or directory, stat '/foo/bar/notxist.csv'",
done,
);
});
});
describe('/:source/:bucket/:key', () => {
it('downloads a minio artifact', done => {
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/minio/ml-pipeline/hello/world.txt') // url
.expect(200, artifactContent, done);
});
it('downloads a tar gzipped artifact as is', done => {
// base64 encoding for tar gzipped 'hello-world'
const tarGzBase64 =
'H4sIAFa7DV4AA+3PSwrCMBRG4Y5dxV1BuSGPridgwcItkTZSl++johNBJ0WE803OIHfwZ87j0fq2nmuzGVVNIcitXYqPpntXLojzSb33MToVdTG5rhHdbtLLaa55uk5ZBrMhj23ty9u7T+/rT+TZP3HozYosZbL97tdbAAAAAAAAAAAAAAAAAADfuwAyiYcHACgAAA==';
const tarGzBuffer = Buffer.from(tarGzBase64, 'base64');
artifactContent = tarGzBuffer;
const configs = loadConfigs(argv, {});
app = new UIServer(configs);
const request = requests(app.start());
request
.get('/artifacts/minio/ml-pipeline/hello/world.txt') // url
.expect(200, tarGzBuffer.toString(), done);
});
});
});

30
frontend/server/integration-tests/artifact-proxy.test.ts
View File

@ -109,6 +109,36 @@ describe('/artifacts/get namespaced proxy', () => {
});
});
it('proxies a download request to namespaced artifact service', done => {
const { receivedUrls, getArtifactServiceGetterSpy } = setUpNamespacedArtifactService({
namespace: 'ns2',
});
const configs = loadConfigs(argv, {
ARTIFACTS_SERVICE_PROXY_NAME: 'artifact-svc',
ARTIFACTS_SERVICE_PROXY_PORT: '80',
ARTIFACTS_SERVICE_PROXY_ENABLED: 'true',
});
app = new UIServer(configs);
requests(app.start())
.get(
`/artifacts/minio/ml-pipeline/hello.txt${buildQuery({
namespace: 'ns2',
})}`,
)
.expect(200, 'artifact service in ns2', err => {
expect(getArtifactServiceGetterSpy).toHaveBeenCalledWith({
serviceName: 'artifact-svc',
servicePort: 80,
enabled: true,
});
expect(receivedUrls).toEqual(
// url is the same, except namespace query is omitted
['/artifacts/minio/ml-pipeline/hello.txt'],
);
done(err);
});
});
it('does not proxy requests without namespace argument', done => {
setupMinioArtifactDeps({ content: 'text-data2' });
const configs = loadConfigs(argv, { ARTIFACTS_SERVICE_PROXY_ENABLED: 'true' });

9
frontend/server/integration-tests/test-helper.ts
View File

@ -3,7 +3,7 @@ import * as os from 'os';
import * as fs from 'fs';
export function commonSetup(
options: { commitHash?: string; tagName?: string } = {},
options: { commitHash?: string; tagName?: string; showLog?: boolean } = {},
): { argv: string[]; buildDate: string; indexHtmlPath: string; indexHtmlContent: string } {
const indexHtmlPath = path.resolve(os.tmpdir(), 'index.html');
const argv = ['node', 'dist/server.js', os.tmpdir(), '3000'];
@ -33,6 +33,13 @@ export function commonSetup(
jest.restoreAllMocks();
});
if (!options.showLog) {
beforeEach(() => {
jest.spyOn(global.console, 'info').mockImplementation();
jest.spyOn(global.console, 'log').mockImplementation();
});
}
return { argv, buildDate, indexHtmlPath, indexHtmlContent };
}

6
frontend/server/minio-helper.ts
View File

@ -1,3 +1,4 @@
import { Stream } from 'stream';
// Copyright 2019-2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -23,6 +24,7 @@ export interface MinioRequestConfig {
bucket: string;
key: string;
client: MinioClient;
tryExtract?: boolean;
}
/** MinioClientOptionsWithOptionalSecrets wraps around MinioClientOptions where only endPoint is required (accesskey and secretkey are optional). */
@ -126,13 +128,15 @@ function extractFirstTarRecordAsStream() {
* @param param.bucket Bucket name to retrieve the object from.
* @param param.key Key of the object to retrieve.
* @param param.client Minio client.
* @param param.tryExtract Whether we try to extract *.tar.gz, default to true.
*
*/
export async function getObjectStream({
bucket,
key,
client,
tryExtract = true,
}: MinioRequestConfig): Promise<Transform> {
const stream = await client.getObject(bucket, key);
return stream.pipe(gunzip()).pipe(maybeTarball());
return tryExtract ? stream.pipe(gunzip()).pipe(maybeTarball()) : stream.pipe(new PassThrough());
}

6
frontend/src/atoms/ExternalLink.tsx
View File

@ -1,6 +1,7 @@
import React, { DetailedHTMLProps, AnchorHTMLAttributes } from 'react';
import { stylesheet } from 'typestyle';
import { color } from '../Css';
import { classes } from 'typestyle';
const css = stylesheet({
link: {
@ -11,6 +12,7 @@ const css = stylesheet({
},
color: color.theme,
textDecoration: 'none',
wordBreak: 'break-all', // Links do not need to break at words.
},
});
@ -19,7 +21,7 @@ export const ExternalLink: React.FC<DetailedHTMLProps<
HTMLAnchorElement
>> = props => (
// eslint-disable-next-line jsx-a11y/anchor-has-content
<a {...props} className={css.link} target='_blank' rel='noopener' />
<a {...props} className={classes(css.link, props.className)} target='_blank' rel='noopener' />
);
export const AutoLink: React.FC<DetailedHTMLProps<
@ -28,7 +30,7 @@ export const AutoLink: React.FC<DetailedHTMLProps<
>> = props =>
props.href && props.href.startsWith('#') ? (
// eslint-disable-next-line jsx-a11y/anchor-has-content
<a {...props} className={css.link} />
<a {...props} className={classes(css.link, props.className)} />
) : (
<ExternalLink {...props} />
);

230
frontend/src/components/MinioArtifactPreview.test.tsx
View File

@ -23,11 +23,8 @@ describe('MinioArtifactPreview', () => {
const readFile = jest.spyOn(Apis, 'readFile');
beforeEach(() => {
readFile.mockResolvedValue('preview ...');
});
afterEach(() => {
jest.resetAllMocks();
readFile.mockResolvedValue('preview ...');
});
it('handles undefined artifact', () => {
@ -103,15 +100,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=s3&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="s3://foo/bar"
<div
class="topDiv"
>
s3://foo/bar
</a>
<a
class="link"
href="artifacts/s3/foo/bar"
rel="noopener"
target="_blank"
title="s3://foo/bar"
>
s3://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=s3&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
<div
class="preview"
>
@ -143,15 +155,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
<div
class="topDiv"
>
minio://foo/bar
</a>
<a
class="link"
href="artifacts/minio/foo/bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
>
minio://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
<div
class="preview"
>
@ -183,15 +210,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=minio&namespace=namespace&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
<div
class="topDiv"
>
minio://foo/bar
</a>
<a
class="link"
href="artifacts/minio/foo/bar?namespace=namespace"
rel="noopener"
target="_blank"
title="minio://foo/bar"
>
minio://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=minio&namespace=namespace&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
<div
class="preview"
>
@ -222,15 +264,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
<div
class="topDiv"
>
minio://foo/bar
</a>
<a
class="link"
href="artifacts/minio/foo/bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
>
minio://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
</div>
</div>
`);
@ -246,7 +303,7 @@ describe('MinioArtifactPreview', () => {
};
const data = `012\n345\n678\n910`;
readFile.mockResolvedValue(data);
const { container } = render(
const { container, queryByText } = render(
<MinioArtifactPreview value={minioArtifact} maxbytes={data.length} />,
);
await act(TestUtils.flushPromises);
@ -255,15 +312,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
<div
class="topDiv"
>
minio://foo/bar
</a>
<a
class="link"
href="artifacts/minio/foo/bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
>
minio://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
<div
class="preview"
>
@ -291,7 +363,7 @@ describe('MinioArtifactPreview', () => {
};
const data = `012\n345\n678\n910`;
readFile.mockResolvedValue(data);
const { container } = render(
const { container, queryByText } = render(
<MinioArtifactPreview value={minioArtifact} maxbytes={data.length} maxlines={2} />,
);
await act(TestUtils.flushPromises);
@ -300,15 +372,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
<div
class="topDiv"
>
minio://foo/bar
</a>
<a
class="link"
href="artifacts/minio/foo/bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
>
minio://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
<div
class="preview"
>
@ -323,6 +410,7 @@ describe('MinioArtifactPreview', () => {
</div>
</div>
`);
expect(queryByText('View All')).toBeTruthy();
});
it('handles artifact that previews with maxbytes', async () => {
@ -335,7 +423,7 @@ describe('MinioArtifactPreview', () => {
};
const data = `012\n345\n678\n910`;
readFile.mockResolvedValue(data);
const { container } = render(
const { container, queryByText } = render(
<MinioArtifactPreview value={minioArtifact} maxbytes={data.length - 5} />,
);
await act(TestUtils.flushPromises);
@ -344,15 +432,30 @@ describe('MinioArtifactPreview', () => {
<div
class="root"
>
<a
class="link"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
<div
class="topDiv"
>
minio://foo/bar
</a>
<a
class="link"
href="artifacts/minio/foo/bar"
rel="noopener"
target="_blank"
title="minio://foo/bar"
>
minio://foo/bar
</a>
<span
class="separater"
/>
<a
class="link viewLink"
href="artifacts/get?source=minio&bucket=foo&key=bar"
rel="noopener"
target="_blank"
>
View All
</a>
</div>
<div
class="preview"
>
@ -368,5 +471,6 @@ describe('MinioArtifactPreview', () => {
</div>
</div>
`);
expect(queryByText('View All')).toBeTruthy();
});
});

55
frontend/src/components/MinioArtifactPreview.tsx
View File

@ -18,6 +18,17 @@ const css = stylesheet({
padding: 3,
backgroundColor: color.lightGrey,
},
topDiv: {
display: 'flex',
justifyContent: 'space-between',
},
separater: {
width: 20, // There's minimum 20px separation between URI and view button.
display: 'inline-block',
},
viewLink: {
whiteSpace: 'nowrap',
},
});
/**
@ -47,12 +58,12 @@ async function getPreview(
namespace: string | undefined,
maxbytes: number,
maxlines?: number,
) {
): Promise<{ data: string; hasMore: boolean }> {
// TODO how to handle binary data (can probably use magic number to id common mime types)
let data = await Apis.readFile(storagePath, namespace, maxbytes + 1);
// is preview === data and no maxlines
if (data.length <= maxbytes && !maxlines) {
return data;
return { data, hasMore: false };
}
// remove extra byte at the end (we requested maxbytes +1)
data = data.slice(0, maxbytes);
@ -64,7 +75,7 @@ async function getPreview(
.join('\n')
.trim();
}
return `${data}\n...`;
return { data: `${data}\n...`, hasMore: true };
}
/**
@ -76,21 +87,26 @@ const MinioArtifactPreview: React.FC<MinioArtifactPreviewProps> = ({
maxbytes = 255,
maxlines,
}) => {
const [content, setContent] = React.useState<string | undefined>(undefined);
const [content, setContent] = React.useState<{ data: string; hasMore: boolean } | undefined>(
undefined,
);
const storagePath = getStoragePath(value);
const source = storagePath?.source;
const bucket = storagePath?.bucket;
const key = storagePath?.key;
React.useEffect(() => {
let cancelled = false;
if (storagePath) {
getPreview(storagePath, namespace, maxbytes, maxlines).then(
data => !cancelled && setContent(data),
if (source && bucket && key) {
getPreview({ source, bucket, key }, namespace, maxbytes, maxlines).then(
content => !cancelled && setContent(content),
error => console.error(error), // TODO error badge on link?
);
}
return () => {
cancelled = true;
};
}, [storagePath, namespace, maxbytes, maxlines]);
}, [source, bucket, key, namespace, maxbytes, maxlines]);
if (!storagePath) {
// if value is undefined, null, or an invalid s3artifact object, don't render
@ -102,19 +118,30 @@ const MinioArtifactPreview: React.FC<MinioArtifactPreviewProps> = ({
// TODO need to come to an agreement how to encode artifact info inside a url
// namespace is currently not supported
const linkText = Apis.buildArtifactUrl(storagePath);
const artifactUrl = Apis.buildReadFileUrl(storagePath, namespace);
const artifactDownloadUrl = Apis.buildReadFileUrl({
path: storagePath,
namespace,
isDownload: true,
});
const artifactViewUrl = Apis.buildReadFileUrl({ path: storagePath, namespace });
// Opens in new window safely
// TODO use ArtifactLink instead (but it need to support namespace)
return (
<div className={css.root}>
<ExternalLink href={artifactUrl} title={linkText}>
{linkText}
</ExternalLink>
{content && (
<div className={css.topDiv}>
<ExternalLink href={artifactDownloadUrl} title={linkText}>
{linkText}
</ExternalLink>
<span className={css.separater}></span>
<ExternalLink href={artifactViewUrl} className={css.viewLink}>
View All
</ExternalLink>
</div>
{content?.data && (
<div className={css.preview}>
<small>
<pre>{content}</pre>
<pre>{content.data}</pre>
</small>
</div>
)}

10
frontend/src/lib/Apis.test.ts
View File

@ -137,15 +137,15 @@ describe('Apis', () => {
it('buildReadFileUrl', () => {
expect(
Apis.buildReadFileUrl(
{
Apis.buildReadFileUrl({
path: {
bucket: 'testbucket',
key: 'testkey',
source: StorageService.GCS,
},
'testnamespace',
255,
),
namespace: 'testnamespace',
peek: 255,
}),
).toEqual(
'artifacts/get?source=gcs&namespace=testnamespace&peek=255&bucket=testbucket&key=testkey',
);

28
frontend/src/lib/Apis.ts
View File

@ -208,16 +208,34 @@ export class Apis {
* Reads file from storage using server.
*/
public static readFile(path: StoragePath, namespace?: string, peek?: number): Promise<string> {
return this._fetch(this.buildReadFileUrl(path, namespace, peek));
return this._fetch(this.buildReadFileUrl({ path, namespace, peek, isDownload: false }));
}
/**
* Builds an url for the readFile API to retrieve a workflow artifact.
* @param props object describing the artifact (e.g. source, bucket, and key)
* @param path object describing the artifact (e.g. source, bucket, and key)
* @param isDownload whether we download the artifact as is (e.g. skip extracting from *.tar.gz)
*/
public static buildReadFileUrl(path: StoragePath, namespace?: string, peek?: number) {
const { source, ...rest } = path;
return `artifacts/get${buildQuery({ source: `${source}`, namespace, peek, ...rest })}`;
public static buildReadFileUrl({
path,
namespace,
peek,
isDownload,
}: {
path: StoragePath;
namespace?: string;
peek?: number;
isDownload?: boolean;
}) {
const { source, bucket, key } = path;
if (isDownload) {
return `artifacts/${source}/${bucket}/${key}${buildQuery({
namespace,
peek,
})}`;
} else {
return `artifacts/get${buildQuery({ source, namespace, peek, bucket, key })}`;
}
}
/**

6
frontend/src/lib/Utils.test.ts
View File

@ -243,7 +243,7 @@ describe('Utils', () => {
describe('generateMinioArtifactUrl', () => {
it('handles minio:// URIs', () => {
expect(generateMinioArtifactUrl('minio://my-bucket/a/b/c')).toBe(
'artifacts/get?source=minio&bucket=my-bucket&key=a%2Fb%2Fc',
'artifacts/minio/my-bucket/a/b/c',
);
});
@ -258,9 +258,7 @@ describe('Utils', () => {
describe('generateS3ArtifactUrl', () => {
it('handles s3:// URIs', () => {
expect(generateS3ArtifactUrl('s3://my-bucket/a/b/c')).toBe(
'artifacts/get?source=s3&bucket=my-bucket&key=a%2Fb%2Fc',
);
expect(generateS3ArtifactUrl('s3://my-bucket/a/b/c')).toBe('artifacts/s3/my-bucket/a/b/c');
});
});

31
frontend/src/lib/Utils.tsx
View File

@ -21,11 +21,12 @@ import { ApiTrigger } from '../apis/job';
import { Workflow } from '../../third_party/argo-ui/argo_template';
import { isFunction } from 'lodash';
import { hasFinished, NodePhase } from './StatusUtils';
import { ListRequest } from './Apis';
import { ListRequest, Apis } from './Apis';
import { Row, Column, ExpandState } from '../components/CustomTable';
import { padding } from '../Css';
import { classes } from 'typestyle';
import { CustomTableRow, css } from '../components/CustomTableRow';
import { StorageService } from './WorkflowParser';
export const logger = {
error: (...args: any[]) => {
@ -293,23 +294,6 @@ export function generateGcsConsoleUri(gcsUri: string): string | undefined {
const MINIO_URI_PREFIX = 'minio://';
/**
* Generates the path component of the url to retrieve an artifact.
*
* @param source source of the artifact. Can be "minio", "s3", "http", "https", or "gcs".
* @param bucket bucket where the artifact is stored, value is assumed to be uri encoded.
* @param key path to the artifact, value is assumed to be uri encoded.
* @param peek number of characters or bytes to return. If not provided, the entire content of the artifact will be returned.
*/
export function generateArtifactUrl(
source: string,
bucket: string,
key: string,
peek?: number,
): string {
return `artifacts/get${buildQuery({ source, bucket, key, peek })}`;
}
/**
* Generates an HTTPS API URL from minio:// uri
*
@ -326,7 +310,11 @@ export function generateMinioArtifactUrl(minioUri: string, peek?: number): strin
if (matches == null) {
return undefined;
}
return generateArtifactUrl('minio', matches[1], matches[2], peek);
return Apis.buildReadFileUrl({
path: { source: StorageService.MINIO, bucket: matches[1], key: matches[2] },
peek,
isDownload: true,
});
}
const S3_URI_PREFIX = 's3://';
@ -346,7 +334,10 @@ export function generateS3ArtifactUrl(s3Uri: string): string | undefined {
if (matches == null) {
return undefined;
}
return generateArtifactUrl('s3', matches[1], matches[2]);
return Apis.buildReadFileUrl({
path: { source: StorageService.S3, bucket: matches[1], key: matches[2] },
isDownload: true,
});
}
export function buildQuery(queriesMap: { [key: string]: string | number | undefined }): string {