[UIServer] Add DISABLE_GKE_METADATA env flag to skip metadata retrieval. (#3118)

* Add DISABLE_GKE_METADATA env flag to skip metadata retrieval.

* node server build step fix and cleanup

* Update frontend dockerfile to use npm ci

frontend/Dockerfile

@@ -9,7 +9,7 @@ COPY . .
 
 WORKDIR ./frontend
 
-RUN npm install && npm run postinstall
+RUN npm ci && npm run postinstall
 RUN npm run build
 
 RUN mkdir -p ./server/dist && \

frontend/package.json

@@ -47,7 +47,7 @@
     "lint": "tslint -c ./tslint.prod.json -p .",
     "mock:api": "ts-node-dev -O '{\"module\": \"commonjs\"}' mock-backend/mock-api-server.ts 3001",
     "mock:server": "node server/dist/server.js build",
-    "postinstall": "cd ./server && npm i && cd ../mock-backend && npm i",
+    "postinstall": "cd ./server && npm ci && cd ../mock-backend && npm ci",
     "start:proxy-standalone": "./start-proxy-standalone.sh",
     "start:proxy-standalone-and-server": "./start-proxy-standalone-and-server.sh",
     "start": "react-scripts-ts start",

frontend/server/app.test.ts

@@ -24,6 +24,7 @@ import { Storage as GCSStorage } from '@google-cloud/storage';
 import { UIServer } from './app';
 import { loadConfigs } from './configs';
 import * as minioHelper from './minio-helper';
+import * as k8sHelper from './k8s-helper';
 
 jest.mock('minio');
 jest.mock('node-fetch');
@@ -31,6 +32,9 @@ jest.mock('@google-cloud/storage');
 jest.mock('./minio-helper');
 jest.mock('./k8s-helper');
 
+const mockedFetch: jest.Mock = fetch as any;
+const mockedK8sHelper: jest.Mock = k8sHelper as any;
+
 describe('UIServer apis', () => {
   let app: UIServer;
   const indexHtmlPath = path.resolve(os.tmpdir(), 'index.html');
@@ -269,7 +273,6 @@ describe('UIServer apis', () => {
 
     it('responds with a http artifact if source=http', done => {
       const artifactContent = 'hello world';
-      const mockedFetch: jest.Mock = fetch as any;
       mockedFetch.mockImplementationOnce((url: string, opts: any) =>
         url === 'http://foo.bar/ml-pipeline/hello/world.txt'
           ? Promise.resolve({ buffer: () => Promise.resolve(artifactContent) })
@@ -293,7 +296,6 @@ describe('UIServer apis', () => {
 
     it('responds with a https artifact if source=https', done => {
       const artifactContent = 'hello world';
-      const mockedFetch: jest.Mock = fetch as any;
       mockedFetch.mockImplementationOnce((url: string, opts: any) =>
         url === 'https://foo.bar/ml-pipeline/hello/world.txt' &&
         opts.headers.Authorization === 'someToken'
@@ -322,7 +324,6 @@ describe('UIServer apis', () => {
 
     it('responds with a https artifact using the inherited header if source=https and http authorization key is provided.', done => {
       const artifactContent = 'hello world';
-      const mockedFetch: jest.Mock = fetch as any;
       mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
         url === 'https://foo.bar/ml-pipeline/hello/world.txt'
           ? Promise.resolve({ buffer: () => Promise.resolve(artifactContent) })
@@ -370,6 +371,65 @@ describe('UIServer apis', () => {
     });
   });
 
+  describe('/system', () => {
+    describe('/cluster-name', () => {
+      it('responds with cluster name data from gke metadata', done => {
+        mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
+          url === 'http://metadata/computeMetadata/v1/instance/attributes/cluster-name'
+            ? Promise.resolve({ text: () => Promise.resolve('test-cluster') })
+            : Promise.reject('Unexpected request'),
+        );
+        mockedK8sHelper.isInCluster = true;
+        const configs = loadConfigs(argv, {});
+        app = new UIServer(configs);
+
+        const request = requests(app.start());
+        request
+          .get('/system/cluster-name')
+          .expect('Content-Type', 'text/html; charset=utf-8')
+          .expect(200, 'test-cluster', done);
+      });
+      it('responds with endpoint disabled if DISABLE_GKE_METADATA env is true', done => {
+        const configs = loadConfigs(argv, { DISABLE_GKE_METADATA: 'true' });
+        app = new UIServer(configs);
+
+        const request = requests(app.start());
+        request
+          .get('/system/cluster-name')
+          .expect('Content-Type', 'text/html; charset=utf-8')
+          .expect(500, 'GKE metadata endpoints are disabled.', done);
+      });
+    });
+    describe('/project-id', () => {
+      it('responds with project id data from gke metadata', done => {
+        mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
+          url === 'http://metadata/computeMetadata/v1/project/project-id'
+            ? Promise.resolve({ text: () => Promise.resolve('test-project') })
+            : Promise.reject('Unexpected request'),
+        );
+        mockedK8sHelper.isInCluster = true;
+        const configs = loadConfigs(argv, {});
+        app = new UIServer(configs);
+
+        const request = requests(app.start());
+        request
+          .get('/system/project-id')
+          .expect('Content-Type', 'text/html; charset=utf-8')
+          .expect(200, 'test-project', done);
+      });
+      it('responds with endpoint disabled if DISABLE_GKE_METADATA env is true', done => {
+        const configs = loadConfigs(argv, { DISABLE_GKE_METADATA: 'true' });
+        app = new UIServer(configs);
+
+        const request = requests(app.start());
+        request
+          .get('/system/project-id')
+          .expect('Content-Type', 'text/html; charset=utf-8')
+          .expect(500, 'GKE metadata endpoints are disabled.', done);
+      });
+    });
+  });
+
   // TODO: refractor k8s helper module so that api that interact with k8s can be
   // mocked and tested. There is currently no way to mock k8s APIs as
   // `k8s-helper.isInCluster` is a constant that is generated when the module is

frontend/server/app.ts

@@ -26,7 +26,7 @@ import {
   deleteTensorboardHandler,
 } from './handlers/tensorboard';
 import { getPodLogsHandler } from './handlers/pod-logs';
-import { clusterNameHandler, projectIdHandler } from './handlers/gke-metadata';
+import { getClusterNameHandler, getProjectIdHandler } from './handlers/gke-metadata';
 import { getAllowCustomVisualizationsHandler } from './handlers/vis';
 import { getIndexHTMLHandler } from './handlers/index-html';
 
@@ -128,8 +128,8 @@ function createUIServer(options: UIConfigs) {
   registerHandler(app.get, '/k8s/pod/logs', getPodLogsHandler(options.argo, options.artifacts));
 
   /** Cluster metadata (GKE only) */
-  registerHandler(app.get, '/system/cluster-name', clusterNameHandler);
-  registerHandler(app.get, '/system/project-id', projectIdHandler);
+  registerHandler(app.get, '/system/cluster-name', getClusterNameHandler(options.gkeMetadata));
+  registerHandler(app.get, '/system/project-id', getProjectIdHandler(options.gkeMetadata));
 
   /** Visualization */
   registerHandler(

frontend/server/configs.ts

@@ -84,6 +84,8 @@ export function loadConfigs(
     ARGO_ARCHIVE_BUCKETNAME = 'mlpipeline',
     /** Prefix to logs. */
     ARGO_ARCHIVE_PREFIX = 'logs',
+    /** Disables GKE metadata endpoint. */
+    DISABLE_GKE_METADATA = 'false',
     /** Deployment type. */
     DEPLOYMENT: DEPLOYMENT_STR = '',
   } = env;
@@ -149,6 +151,9 @@ export function loadConfigs(
     visualizations: {
       allowCustomVisualizations: asBool(ALLOW_CUSTOM_VISUALIZATIONS),
     },
+    gkeMetadata: {
+      disabled: asBool(DISABLE_GKE_METADATA),
+    },
   };
 }
 
@@ -202,6 +207,9 @@ export interface ServerConfigs {
   apiVersionPrefix: string;
   deployment: Deployments;
 }
+export interface GkeMetadataConfigs {
+  disabled: boolean;
+}
 export interface UIConfigs {
   server: ServerConfigs;
   artifacts: {
@@ -214,4 +222,5 @@ export interface UIConfigs {
   visualizations: VisualizationsConfigs;
   viewer: ViewerConfigs;
   pipeline: PipelineConfigs;
+  gkeMetadata: GkeMetadataConfigs;
 }

frontend/server/handlers/gke-metadata.ts

@@ -14,8 +14,20 @@
 import { Handler } from 'express';
 import * as k8sHelper from '../k8s-helper';
 import fetch from 'node-fetch';
+import { GkeMetadataConfigs } from '../configs';
 
-export const clusterNameHandler: Handler = async (_, res) => {
+const disabledHandler: Handler = async (_, res) => {
+  res.status(500).send('GKE metadata endpoints are disabled.');
+};
+
+export const getClusterNameHandler = (options: GkeMetadataConfigs) => {
+  if (options.disabled) {
+    return disabledHandler;
+  }
+  return clusterNameHandler;
+};
+
+const clusterNameHandler: Handler = async (_, res) => {
   if (!k8sHelper.isInCluster) {
     res.status(500).send('Not running in Kubernetes cluster.');
     return;
@@ -28,7 +40,14 @@ export const clusterNameHandler: Handler = async (_, res) => {
   res.send(await response.text());
 };
 
-export const projectIdHandler: Handler = async (_, res) => {
+export const getProjectIdHandler = (options: GkeMetadataConfigs) => {
+  if (options.disabled) {
+    return disabledHandler;
+  }
+  return projectIdHandler;
+};
+
+const projectIdHandler: Handler = async (_, res) => {
   if (!k8sHelper.isInCluster) {
     res.status(500).send('Not running in Kubernetes cluster.');
     return;

frontend/server/package.json

@@ -16,20 +16,20 @@
   "devDependencies": {
     "@types/crypto-js": "^3.1.43",
     "@types/express": "^4.11.1",
-    "@types/jest": "^24.0.23",
+    "@types/jest": "^24.9.1",
     "@types/minio": "^7.0.3",
     "@types/node": "^10.17.11",
     "@types/node-fetch": "^2.1.2",
     "@types/supertest": "^2.0.8",
     "@types/tar": "^4.0.3",
-    "jest": "^24.9.0",
+    "jest": "^25.1.0",
     "supertest": "^4.0.2",
-    "ts-jest": "^24.2.0",
+    "ts-jest": "^25.2.1",
     "tslint": "^5.20.1",
     "typescript": "^3.6.4"
   },
   "scripts": {
-    "build": "tsc --lib es2015 --outDir ./dist --typeRoots ./node_modules/@types *.ts",
+    "build": "tsc --lib es2015 --outDir ./dist --typeRoots ./node_modules/@types server.ts",
     "format": "npx prettier --write './**/*.{ts,tsx}'",
     "format:check": "npx prettier --check './**/*.{ts,tsx}' || node ../scripts/check-format-error-info.js",
     "lint": "npx tslint -c ../tslint.prod.json -p ../tsconfig.prod.json",
@@ -45,7 +45,11 @@
         "diagnostics": false,
         "tsConfig": "../tsconfig.json"
       }
-    }
+    },
+    "testPathIgnorePatterns": [
+      "/node_modules/",
+      "dist/"
+    ]
   },
   "repository": {
     "type": "git",