package integration import ( "testing" "encoding/json" "github.com/golang/glog" "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_model" "github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_model" "github.com/kubeflow/pipelines/backend/src/cmd/ml/cmd" "github.com/kubeflow/pipelines/backend/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) const ( defaultPageSize = int32(10) myUploadedPipeline = "my-uploaded-pipeline" ) func GetRealRootCommand() (*cmd.RootCommand, cmd.ClientFactoryInterface) { clientFactory := cmd.NewClientFactoryWithByteBuffer() rootCmd := cmd.NewRootCmd(clientFactory) rootCmd = cmd.CreateSubCommands(rootCmd, defaultPageSize) return rootCmd, clientFactory } type CLIIntegrationTest struct { suite.Suite namespace string } // Check the cluster namespace has Kubeflow pipelines installed and ready. func (c *CLIIntegrationTest) SetupTest() { if !*runIntegrationTests { c.T().SkipNow() return } c.namespace = *namespace // Wait for the system to be ready. err := test.WaitForReady(c.namespace, *initializeTimeout) if err != nil { glog.Exitf("Cluster namespace '%s' is still not ready after timeout. Error: %s", c.namespace, err.Error()) } } func (c *CLIIntegrationTest) TearDownTest() { err := c.deletePipelines() if err != nil { glog.Exitf("Failed to delete pipelines: %v", err) } } func (c *CLIIntegrationTest) TestPipelineListSuccess() { t := c.T() // Create pipelines rootCmd, _ := GetRealRootCommand() args := []string{"pipeline", "create", "--url", "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml"} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err := rootCmd.Command().ExecuteC() assert.Nil(t, err) // List pipeline rootCmd, factory := GetRealRootCommand() args = []string{"pipeline", "list"} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err = rootCmd.Command().ExecuteC() assert.Nil(t, err) // Convert and assert result pipelines, err := toPipelines(factory.Result()) assert.Nil(t, err) assert.True(t, len(pipelines) >= 1) } func (c *CLIIntegrationTest) TestPipelineListFailureInvalidArgument() { t := c.T() rootCmd, _ := GetRealRootCommand() args := []string{"pipeline", "list", "EXTRA_ARGUMENT"} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err := rootCmd.Command().ExecuteC() assert.NotNil(t, err) } func (c *CLIIntegrationTest) TestPipelineUploadSuccess() { t := c.T() rootCmd, _ := GetRealRootCommand() args := []string{"pipeline", "upload", "--name", myUploadedPipeline, "--file", "../resources/hello-world.yaml"} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err := rootCmd.Command().ExecuteC() assert.Nil(t, err) } func (c *CLIIntegrationTest) TestPipelineCreateGetDeleteSuccess() { t := c.T() rootCmd, factory := GetRealRootCommand() // Create pipeline args := []string{"pipeline", "create", "--name", "foo", "--url", "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml"} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err := rootCmd.Command().ExecuteC() assert.Nil(t, err) // Get ID pipelineID, err := toPipelineID(factory.Result()) assert.Nil(t, err) // Get pipeline args = []string{"pipeline", "get", "--id", pipelineID} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err = rootCmd.Command().ExecuteC() assert.Nil(t, err) // Get manifest args = []string{"pipeline", "get-manifest", "--id", pipelineID} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err = rootCmd.Command().ExecuteC() assert.Nil(t, err) // Delete pipeline args = []string{"pipeline", "delete", "--id", pipelineID} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err = rootCmd.Command().ExecuteC() assert.Nil(t, err) } // TODO: uncomment these tests once the "Delete Experiment" API is available. //func (c *CLIIntegrationTest) TestExperimentListSuccess() { // t := c.T() // // // Create pipelines // rootCmd, _ := GetRealRootCommand() // args := []string{"experiment", "create", "--name", // "MY_EXPERIMENT"} // args = addCommonArgs(args, c.namespace) // rootCmd.Command().SetArgs(args) // _, err := rootCmd.Command().ExecuteC() // assert.Nil(t, err) // // // List pipeline // rootCmd, factory := GetRealRootCommand() // args = []string{"pipeline", "list"} // args = addCommonArgs(args, c.namespace) // rootCmd.Command().SetArgs(args) // _, err = rootCmd.Command().ExecuteC() // assert.Nil(t, err) // // // Convert and assert result // pipelines, err := toExperiments(factory.Result()) // assert.Nil(t, err) // assert.True(t, len(pipelines) >= 1) //} // //func (c *CLIIntegrationTest) TestExperimentListFailureInvalidArgument() { // t := c.T() // rootCmd, _ := GetRealRootCommand() // args := []string{"experiment", "list", "EXTRA_ARGUMENT"} // args = addCommonArgs(args, c.namespace) // rootCmd.Command().SetArgs(args) // _, err := rootCmd.Command().ExecuteC() // assert.NotNil(t, err) //} // //func (c *CLIIntegrationTest) TestExperimentCreateGetSuccess() { // t := c.T() // rootCmd, factory := GetRealRootCommand() // // // Create pipeline // args := []string{"experiment", "create", "--name", // "MY_EXPERIMENT"} // args = addCommonArgs(args, c.namespace) // rootCmd.Command().SetArgs(args) // _, err := rootCmd.Command().ExecuteC() // assert.Nil(t, err) // // // Get ID // experimentID, err := toExperimentID(factory.Result()) // assert.Nil(t, err) // // // Get pipeline // args = []string{"experiment", "get", "--id", experimentID} // args = addCommonArgs(args, c.namespace) // rootCmd.Command().SetArgs(args) // _, err = rootCmd.Command().ExecuteC() // assert.Nil(t, err) //} func TestCLI(t *testing.T) { suite.Run(t, new(CLIIntegrationTest)) } func addCommonArgs(args []string, namespace string) []string { args = append(args, "--debug", "--namespace", namespace, "-o", "json") return args } func toPipelineID(jsonPipeline string) (string, error) { var pipeline pipeline_model.APIPipeline err := json.Unmarshal([]byte(jsonPipeline), &pipeline) if err != nil { return "", err } return pipeline.ID, nil } func toPipelines(jsonPipelines string) ([]pipeline_model.APIPipeline, error) { var pipelines []pipeline_model.APIPipeline err := json.Unmarshal([]byte(jsonPipelines), &pipelines) if err != nil { return nil, err } return pipelines, nil } func toExperimentID(jsonExperiment string) (string, error) { var experiment experiment_model.APIExperiment err := json.Unmarshal([]byte(jsonExperiment), &experiment) if err != nil { return "", err } return experiment.ID, nil } func toExperiments(jsonExperiments string) ([]experiment_model.APIExperiment, error) { var experiments []experiment_model.APIExperiment err := json.Unmarshal([]byte(jsonExperiments), &experiments) if err != nil { return nil, err } return experiments, nil } func (c *CLIIntegrationTest) deletePipelines() error { pipelines, err := c.listPipelines() if err != nil { return err } for _, pipeline := range pipelines { err = c.deletePipeline(pipeline) if err != nil { return err } } return nil } func (c *CLIIntegrationTest) deletePipeline(pipeline pipeline_model.APIPipeline) error { rootCmd, _ := GetRealRootCommand() args := []string{"pipeline", "delete", "--id", pipeline.ID} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err := rootCmd.Command().ExecuteC() return err } func (c *CLIIntegrationTest) listPipelines() ([]pipeline_model.APIPipeline, error) { rootCmd, factory := GetRealRootCommand() args := []string{"pipeline", "list"} args = addCommonArgs(args, c.namespace) rootCmd.Command().SetArgs(args) _, err := rootCmd.Command().ExecuteC() if err != nil { return nil, err } return toPipelines(factory.Result()) }