chore(frontend): Create script to build pipeline_spec with protobuf.js. Fix #6205 (#6206)

* chore(frontend): Create script to build pipeline_spec with protobuf.js

* fix generated location

* add Todo about shell script

* add readme for generating pipeline spec

frontend/README.md

@@ -189,8 +189,25 @@ package from
 [google/ml-metadata](https://github.com/google/ml-metadata/tree/master/ml_metadata/proto).
 
 
+## Pipeline Spec (IR) API
+
+For KFP v2, we use pipeline spec or IR(Intermediate Representation) to represent a Pipeline defintion. It is saved as json payload when transmitted. You can find the API in [api/v2alpha1/pipeline_spec.proto](api/v2alpha1/pipeline_spec.proto). To take the latest of this file and compile it to Typescript classes, run the following command under `frontend/` folder:
+
+```
+npm run build:pipeline-spec
+```
+
+Alternatively, you can run the following commands:
+
+```
+npx pbjs -t static-module -w commonjs -o src/generated/pipeline_spec/pbjs_ml_pipelines.js ../api/v2alpha1/pipeline_spec.proto
+npx pbts -o src/generated/pipeline_spec/pbjs_ml_pipelines.d.ts src/generated/pipeline_spec/pbjs_ml_pipelines.js 
+```
+
+You can check out the result in [frontend/src/generated/pipeline_spec](frontend/src/generated/pipeline_spec).
+
 ## Large features development
 
 To accommodate KFP v2 development, we create a `frontend feature flag` capability which hides features under development behind a flag. Only when developer explicitly enables these flags, they can see those features. To control the visiblity of these features, check out a webpage similar to pattern http://localhost:3000/#/frontend_features. 
 
-To manage feature flags default values, visit [frontend/src/feature.ts](frontend/src/feature.ts) for `const features`. To apply the defautl feature flags locally in your browser, run `localStorage.setItem('flags', "")` in browser console.
+To manage feature flags default values, visit [frontend/src/feature.ts](frontend/src/feature.ts) for `const features`. To apply the default feature flags locally in your browser, run `localStorage.setItem('flags', "")` in browser console.

frontend/package-lock.json

@@ -20386,9 +20386,9 @@
       },
       "dependencies": {
         "@types/node": {
-          "version": "16.4.1",
+          "version": "16.4.10",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-16.4.1.tgz",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-16.4.10.tgz",
-          "integrity": "sha512-UW7cbLqf/Wu5XH2RKKY1cHwUNLicIDRLMraYKz+HHAerJ0ZffUEk+fMnd8qU2JaS6cAy0r8tsaf7yqHASf/Y0Q=="
+          "integrity": "sha512-TmVHsm43br64js9BqHWqiDZA+xMtbUpI1MBIA0EyiBmoV9pcEYFOSdj5fr6enZNfh4fChh+AGOLIzGwJnkshyQ=="
         }
       }
     },

frontend/package.json

@@ -22,7 +22,7 @@
     "lodash.isfunction": "^3.0.9",
     "markdown-to-jsx": "^6.10.3",
     "portable-fetch": "^3.0.0",
-    "protobufjs": "^6.11.2",
+    "protobufjs": "~6.11.2",
     "re-resizable": "^4.9.0",
     "react": "^16.12.0",
     "react-ace": "^7.0.2",
@@ -52,6 +52,7 @@
     "build": "npm run lint && EXTEND_ESLINT=true react-scripts build",
     "build:protos": "cd .. && node frontend/scripts/gen_grpc_web_protos.js",
     "build:replace": "node scripts/replace_protos.js",
+    "build:pipeline-spec": "cd .. && node frontend/scripts/pipelinespec_pbjs.js",
     "docker": "COMMIT_HASH=`git rev-parse HEAD`; docker build -q -t ml-pipelines-frontend:${COMMIT_HASH} --build-arg COMMIT_HASH=${COMMIT_HASH} --build-arg DATE=\"`date -u`\" -f Dockerfile ..",
     "eject": "react-scripts eject",
     "format": "prettier --write './**/*.{ts,tsx}'",

frontend/scripts/pipelinespec_pbjs.js

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2021 The Kubeflow Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Note: This script uses protobuf.js to generate pipeline_spec classes in Typescript.
+// Reason: protoc doesn't decode from plain json payload to proto object.
+//         protobuf.js supports this feature along with other protobuf functionality.
+// Usage: Under frontend/, run: `npm run build:pipeline-spec`.
+// Output: Result is stored in frontend/src/third_party/pipeline_spec.
+
+// Explaination of protobufjs cli tool:
+// Install protobufjs-cli by using the main library
+// Command: npm install protobufjs --save --save-prefix=~
+// In the future, this cli will have its own distribution, and isolate from main library.
+
+// Alternatively, run commandline:
+// npx pbjs -t static-module -w commonjs -o src/generated/pipeline_spec/pbjs_ml_pipelines.js ../api/v2alpha1/pipeline_spec.proto
+// npx pbts -o src/generated/pipeline_spec/pbjs_ml_pipelines.d.ts src/generated/pipeline_spec/pbjs_ml_pipelines.js 
+// TODO: Change this file to a shell script for simpler workflow.
+const { spawn } = require('child_process');
+
+(async () => {
+  const sourceProto = 'api/v2alpha1/pipeline_spec.proto';
+  const jsFile = 'frontend/src/generated/pipeline_spec/pbjs_ml_pipelines.js';
+  const tsDefFile = 'frontend/src/generated/pipeline_spec/pbjs_ml_pipelines.d.ts';
+
+  // Generate static javascript file for pipeline_spec.
+  const pbjsProcess = spawn(
+    'pbjs',
+    [`-t static-module`, `-w commonjs`, `-o ${jsFile}`, `${sourceProto}`],
+    {
+      // Allow wildcards in glob to be interpreted
+      shell: true,
+    },
+  );
+  pbjsProcess.stdout.on('data', buffer => console.log(buffer.toString()));
+  pbjsProcess.stderr.on('data', buffer => console.error(buffer.toString()));
+  const code = await new Promise((resolve, reject) => {
+    pbjsProcess.on('error', ex => reject(ex));
+    pbjsProcess.on('close', code => resolve(code));
+  });
+  if (code) {
+    console.log(`Failed to generate ${jsFile}, error code ${code}`);
+    return;
+  }
+  console.log(`${jsFile} succesfully generated.`);
+
+  // Generate typescript definition file based on javascript file.
+  const pbtsProcess = spawn('pbts', [`-o ${tsDefFile}`, `${jsFile}`], {
+    // Allow wildcards in glob to be interpreted
+    shell: true,
+  });
+  pbtsProcess.stdout.on('data', buffer => console.log(buffer.toString()));
+  pbtsProcess.stderr.on('data', buffer => console.error(buffer.toString()));
+  pbtsProcess.on('close', code => {
+    if (code) {
+      console.log(`Failed to generate ${tsDefFile}, error code ${code}`);
+      return;
+    }
+    console.log(`${tsDefFile} succesfully generated.`);
+  });
+})();