kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: Add v2 API integration tests (#9185) 

* Add v2 API integration tests

* move v1 client
This commit is contained in:
Chen Sun 2023-04-27 13:20:39 -07:00 committed by GitHub
parent 6fd71313f6
commit 8b2fba3f39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 4485 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/common/client/api_server/util.go
View File

@ -20,9 +20,9 @@ import (
)
const (
APIServerDefaultTimeout = 35 * time.Second
apiServerBasePath = "/api/v1/namespaces/%s/services/ml-pipeline:8888/proxy/"
apiServerKubeflowInClusterBasePath = "ml-pipeline.%s:8888"
apiServerDefaultTimeout = 35 * time.Second
saDefaultTokenPath = "/var/run/secrets/kubeflow/pipelines/token"
saTokenPathEnvVar = "KF_PIPELINES_SA_TOKEN_PATH"
)

45
backend/src/common/client/api_server/experiment_client.go → backend/src/common/client/api_server/v1/experiment_client.go
View File

@ -22,6 +22,7 @@ import (
apiclient "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/experiment_client"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/experiment_client/experiment_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/experiment_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -45,7 +46,7 @@ type ExperimentClient struct {
func NewExperimentClient(clientConfig clientcmd.ClientConfig, debug bool) (
*ExperimentClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating experiment client: %w", err)
}
@ -55,28 +56,28 @@ func NewExperimentClient(clientConfig clientcmd.ClientConfig, debug bool) (
// Creating experiment client
return &ExperimentClient{
apiClient: apiClient,
authInfoWriter: PassThroughAuth,
authInfoWriter: api_server.PassThroughAuth,
}, nil
}
func NewKubeflowInClusterExperimentClient(namespace string, debug bool) (
*ExperimentClient, error) {
runtime := NewKubeflowInClusterHTTPRuntime(namespace, debug)
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating experiment client
return &ExperimentClient{
apiClient: apiClient,
authInfoWriter: SATokenVolumeProjectionAuth,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *ExperimentClient) Create(parameters *params.CreateExperimentV1Params) (*model.APIExperiment,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -84,9 +85,9 @@ func (c *ExperimentClient) Create(parameters *params.CreateExperimentV1Params) (
response, err := c.apiClient.ExperimentService.CreateExperimentV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.CreateExperimentV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -100,7 +101,7 @@ func (c *ExperimentClient) Create(parameters *params.CreateExperimentV1Params) (
func (c *ExperimentClient) Get(parameters *params.GetExperimentV1Params) (*model.APIExperiment,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -108,9 +109,9 @@ func (c *ExperimentClient) Get(parameters *params.GetExperimentV1Params) (*model
response, err := c.apiClient.ExperimentService.GetExperimentV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetExperimentV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -124,7 +125,7 @@ func (c *ExperimentClient) Get(parameters *params.GetExperimentV1Params) (*model
func (c *ExperimentClient) List(parameters *params.ListExperimentsV1Params) (
[]*model.APIExperiment, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -132,9 +133,9 @@ func (c *ExperimentClient) List(parameters *params.ListExperimentsV1Params) (
response, err := c.apiClient.ExperimentService.ListExperimentsV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListExperimentsV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
@ -147,7 +148,7 @@ func (c *ExperimentClient) List(parameters *params.ListExperimentsV1Params) (
func (c *ExperimentClient) Delete(parameters *params.DeleteExperimentV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -155,9 +156,9 @@ func (c *ExperimentClient) Delete(parameters *params.DeleteExperimentV1Params) e
_, err := c.apiClient.ExperimentService.DeleteExperimentV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DeleteExperimentV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -200,7 +201,7 @@ func listAllForExperiment(client ExperimentInterface, parameters *params.ListExp
func (c *ExperimentClient) Archive(parameters *params.ArchiveExperimentV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -209,9 +210,9 @@ func (c *ExperimentClient) Archive(parameters *params.ArchiveExperimentV1Params)
if err != nil {
if defaultError, ok := err.(*params.ArchiveExperimentV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -224,7 +225,7 @@ func (c *ExperimentClient) Archive(parameters *params.ArchiveExperimentV1Params)
func (c *ExperimentClient) Unarchive(parameters *params.UnarchiveExperimentV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -233,9 +234,9 @@ func (c *ExperimentClient) Unarchive(parameters *params.UnarchiveExperimentV1Par
if err != nil {
if defaultError, ok := err.(*params.UnarchiveExperimentV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,

0
backend/src/common/client/api_server/experiment_client_fake.go → backend/src/common/client/api_server/v1/experiment_client_fake.go
View File

25
backend/src/common/client/api_server/healthz_client.go → backend/src/common/client/api_server/v1/healthz_client.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (
@ -7,6 +21,7 @@ import (
apiclient "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/healthz_client"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/healthz_client/healthz_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/healthz_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"k8s.io/client-go/tools/clientcmd"
)
@ -20,7 +35,7 @@ type HealthzClient struct {
}
func NewHealthzClient(clientConfig clientcmd.ClientConfig, debug bool) (*HealthzClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, err
}
@ -34,13 +49,13 @@ func NewHealthzClient(clientConfig clientcmd.ClientConfig, debug bool) (*Healthz
}
func (c *HealthzClient) GetHealthz() (*model.APIGetHealthzResponse, error) {
parameters := params.NewGetHealthzParamsWithTimeout(apiServerDefaultTimeout)
response, err := c.apiClient.HealthzService.GetHealthz(parameters, PassThroughAuth)
parameters := params.NewGetHealthzParamsWithTimeout(api_server.APIServerDefaultTimeout)
response, err := c.apiClient.HealthzService.GetHealthz(parameters, api_server.PassThroughAuth)
if err != nil {
if defaultError, ok := err.(*params.GetHealthzDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,

57
backend/src/common/client/api_server/job_client.go → backend/src/common/client/api_server/v1/job_client.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (
@ -8,6 +22,7 @@ import (
apiclient "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/job_client"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/job_client/job_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/job_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -32,7 +47,7 @@ type JobClient struct {
func NewJobClient(clientConfig clientcmd.ClientConfig, debug bool) (
*JobClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating job client: %w", err)
}
@ -48,21 +63,21 @@ func NewJobClient(clientConfig clientcmd.ClientConfig, debug bool) (
func NewKubeflowInClusterJobClient(namespace string, debug bool) (
*JobClient, error) {
runtime := NewKubeflowInClusterHTTPRuntime(namespace, debug)
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating job client
return &JobClient{
apiClient: apiClient,
authInfoWriter: SATokenVolumeProjectionAuth,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *JobClient) Create(parameters *params.CreateJobParams) (*model.APIJob,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -70,9 +85,9 @@ func (c *JobClient) Create(parameters *params.CreateJobParams) (*model.APIJob,
response, err := c.apiClient.JobService.CreateJob(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.CreateJobDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -86,7 +101,7 @@ func (c *JobClient) Create(parameters *params.CreateJobParams) (*model.APIJob,
func (c *JobClient) Get(parameters *params.GetJobParams) (*model.APIJob,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -94,9 +109,9 @@ func (c *JobClient) Get(parameters *params.GetJobParams) (*model.APIJob,
response, err := c.apiClient.JobService.GetJob(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetJobDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -109,7 +124,7 @@ func (c *JobClient) Get(parameters *params.GetJobParams) (*model.APIJob,
func (c *JobClient) Delete(parameters *params.DeleteJobParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -117,9 +132,9 @@ func (c *JobClient) Delete(parameters *params.DeleteJobParams) error {
_, err := c.apiClient.JobService.DeleteJob(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DeleteJobDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -132,7 +147,7 @@ func (c *JobClient) Delete(parameters *params.DeleteJobParams) error {
func (c *JobClient) Enable(parameters *params.EnableJobParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -140,9 +155,9 @@ func (c *JobClient) Enable(parameters *params.EnableJobParams) error {
_, err := c.apiClient.JobService.EnableJob(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.EnableJobDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -155,7 +170,7 @@ func (c *JobClient) Enable(parameters *params.EnableJobParams) error {
func (c *JobClient) Disable(parameters *params.DisableJobParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -163,9 +178,9 @@ func (c *JobClient) Disable(parameters *params.DisableJobParams) error {
_, err := c.apiClient.JobService.DisableJob(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DisableJobDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -179,7 +194,7 @@ func (c *JobClient) Disable(parameters *params.DisableJobParams) error {
func (c *JobClient) List(parameters *params.ListJobsParams) (
[]*model.APIJob, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -187,9 +202,9 @@ func (c *JobClient) List(parameters *params.ListJobsParams) (
response, err := c.apiClient.JobService.ListJobs(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListJobsDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,

14
backend/src/common/client/api_server/job_client_fake.go → backend/src/common/client/api_server/v1/job_client_fake.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (

75
backend/src/common/client/api_server/pipeline_client.go → backend/src/common/client/api_server/v1/pipeline_client.go
View File

@ -1,4 +1,4 @@
// Copyright 2018 The Kubeflow Authors
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -23,6 +23,7 @@ import (
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_client/pipeline_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_model"
"github.com/kubeflow/pipelines/backend/src/apiserver/template"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -47,16 +48,16 @@ type PipelineClient struct {
func (c *PipelineClient) UpdateDefaultVersion(parameters *params.UpdatePipelineDefaultVersionV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.PipelineService.UpdatePipelineDefaultVersionV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetPipelineV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -70,7 +71,7 @@ func (c *PipelineClient) UpdateDefaultVersion(parameters *params.UpdatePipelineD
func NewPipelineClient(clientConfig clientcmd.ClientConfig, debug bool) (
*PipelineClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating pipeline client: %w", err)
}
@ -86,30 +87,30 @@ func NewPipelineClient(clientConfig clientcmd.ClientConfig, debug bool) (
func NewKubeflowInClusterPipelineClient(namespace string, debug bool) (
*PipelineClient, error) {
runtime := NewKubeflowInClusterHTTPRuntime(namespace, debug)
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating pipeline client
return &PipelineClient{
apiClient: apiClient,
authInfoWriter: SATokenVolumeProjectionAuth,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *PipelineClient) Create(parameters *params.CreatePipelineV1Params) (*model.APIPipeline,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
parameters.Context = ctx
response, err := c.apiClient.PipelineService.CreatePipelineV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.CreatePipelineV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -123,7 +124,7 @@ func (c *PipelineClient) Create(parameters *params.CreatePipelineV1Params) (*mod
func (c *PipelineClient) Get(parameters *params.GetPipelineV1Params) (*model.APIPipeline,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -131,9 +132,9 @@ func (c *PipelineClient) Get(parameters *params.GetPipelineV1Params) (*model.API
response, err := c.apiClient.PipelineService.GetPipelineV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetPipelineV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -146,7 +147,7 @@ func (c *PipelineClient) Get(parameters *params.GetPipelineV1Params) (*model.API
func (c *PipelineClient) Delete(parameters *params.DeletePipelineV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -154,9 +155,9 @@ func (c *PipelineClient) Delete(parameters *params.DeletePipelineV1Params) error
_, err := c.apiClient.PipelineService.DeletePipelineV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DeletePipelineV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -169,7 +170,7 @@ func (c *PipelineClient) Delete(parameters *params.DeletePipelineV1Params) error
func (c *PipelineClient) DeletePipelineVersion(parameters *params.DeletePipelineVersionV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -177,9 +178,9 @@ func (c *PipelineClient) DeletePipelineVersion(parameters *params.DeletePipeline
_, err := c.apiClient.PipelineService.DeletePipelineVersionV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DeletePipelineVersionV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -191,7 +192,7 @@ func (c *PipelineClient) DeletePipelineVersion(parameters *params.DeletePipeline
func (c *PipelineClient) GetTemplate(parameters *params.GetTemplateParams) (template.Template, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -199,9 +200,9 @@ func (c *PipelineClient) GetTemplate(parameters *params.GetTemplateParams) (temp
response, err := c.apiClient.PipelineService.GetTemplate(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetTemplateDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -216,7 +217,7 @@ func (c *PipelineClient) GetTemplate(parameters *params.GetTemplateParams) (temp
func (c *PipelineClient) List(parameters *params.ListPipelinesV1Params) (
[]*model.APIPipeline, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -224,9 +225,9 @@ func (c *PipelineClient) List(parameters *params.ListPipelinesV1Params) (
response, err := c.apiClient.PipelineService.ListPipelinesV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListPipelinesV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
@ -270,16 +271,16 @@ func listAllForPipeline(client PipelineInterface, parameters *params.ListPipelin
func (c *PipelineClient) CreatePipelineVersion(parameters *params.CreatePipelineVersionV1Params) (*model.APIPipelineVersion,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
parameters.Context = ctx
response, err := c.apiClient.PipelineService.CreatePipelineVersionV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.CreatePipelineVersionV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -293,7 +294,7 @@ func (c *PipelineClient) CreatePipelineVersion(parameters *params.CreatePipeline
func (c *PipelineClient) ListPipelineVersions(parameters *params.ListPipelineVersionsV1Params) (
[]*model.APIPipelineVersion, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -301,9 +302,9 @@ func (c *PipelineClient) ListPipelineVersions(parameters *params.ListPipelineVer
response, err := c.apiClient.PipelineService.ListPipelineVersionsV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListPipelineVersionsV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
@ -317,7 +318,7 @@ func (c *PipelineClient) ListPipelineVersions(parameters *params.ListPipelineVer
func (c *PipelineClient) GetPipelineVersion(parameters *params.GetPipelineVersionV1Params) (*model.APIPipelineVersion,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -325,9 +326,9 @@ func (c *PipelineClient) GetPipelineVersion(parameters *params.GetPipelineVersio
response, err := c.apiClient.PipelineService.GetPipelineVersionV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetPipelineVersionV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -341,7 +342,7 @@ func (c *PipelineClient) GetPipelineVersion(parameters *params.GetPipelineVersio
func (c *PipelineClient) GetPipelineVersionTemplate(parameters *params.GetPipelineVersionTemplateParams) (
template.Template, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -349,9 +350,9 @@ func (c *PipelineClient) GetPipelineVersionTemplate(parameters *params.GetPipeli
response, err := c.apiClient.PipelineService.GetPipelineVersionTemplate(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetPipelineVersionTemplateDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,

15
backend/src/common/client/api_server/pipeline_client_fake.go → backend/src/common/client/api_server/v1/pipeline_client_fake.go
View File

@ -1,7 +1,22 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (
"fmt"
"github.com/kubeflow/pipelines/backend/src/apiserver/template"
"path"

33
backend/src/common/client/api_server/pipeline_upload_client.go → backend/src/common/client/api_server/v1/pipeline_upload_client.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (
@ -10,6 +24,7 @@ import (
apiclient "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/tools/clientcmd"
@ -35,7 +50,7 @@ type PipelineUploadClient struct {
func NewPipelineUploadClient(clientConfig clientcmd.ClientConfig, debug bool) (
*PipelineUploadClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating pipeline upload client: %w", err)
}
@ -51,14 +66,14 @@ func NewPipelineUploadClient(clientConfig clientcmd.ClientConfig, debug bool) (
func NewKubeflowInClusterPipelineUploadClient(namespace string, debug bool) (
*PipelineUploadClient, error) {
runtime := NewKubeflowInClusterHTTPRuntime(namespace, debug)
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating upload client
return &PipelineUploadClient{
apiClient: apiClient,
authInfoWriter: SATokenVolumeProjectionAuth,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
@ -78,7 +93,7 @@ func (c *PipelineUploadClient) UploadFile(filePath string, parameters *params.Up
func (c *PipelineUploadClient) Upload(parameters *params.UploadPipelineParams) (*model.APIPipeline,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -87,9 +102,9 @@ func (c *PipelineUploadClient) Upload(parameters *params.UploadPipelineParams) (
if err != nil {
if defaultError, ok := err.(*params.UploadPipelineDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
@ -113,7 +128,7 @@ func (c *PipelineUploadClient) UploadPipelineVersion(filePath string, parameters
parameters.Uploadfile = runtime.NamedReader(filePath, file)
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -122,9 +137,9 @@ func (c *PipelineUploadClient) UploadPipelineVersion(filePath string, parameters
if err != nil {
if defaultError, ok := err.(*params.UploadPipelineVersionDefault); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,

14
backend/src/common/client/api_server/pipeline_upload_client_fake.go → backend/src/common/client/api_server/v1/pipeline_upload_client_fake.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (

59
backend/src/common/client/api_server/run_client.go → backend/src/common/client/api_server/v1/run_client.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (
@ -9,6 +23,7 @@ import (
apiclient "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client/run_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -33,7 +48,7 @@ type RunClient struct {
func NewRunClient(clientConfig clientcmd.ClientConfig, debug bool) (
*RunClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating run client: %w", err)
}
@ -49,21 +64,21 @@ func NewRunClient(clientConfig clientcmd.ClientConfig, debug bool) (
func NewKubeflowInClusterRunClient(namespace string, debug bool) (
*RunClient, error) {
runtime := NewKubeflowInClusterHTTPRuntime(namespace, debug)
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating run client
return &RunClient{
apiClient: apiClient,
authInfoWriter: SATokenVolumeProjectionAuth,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *RunClient) Create(parameters *params.CreateRunV1Params) (*model.APIRunDetail,
*workflowapi.Workflow, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -71,9 +86,9 @@ func (c *RunClient) Create(parameters *params.CreateRunV1Params) (*model.APIRunD
response, err := c.apiClient.RunService.CreateRunV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetRunV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, nil, util.NewUserError(err,
@ -97,7 +112,7 @@ func (c *RunClient) Create(parameters *params.CreateRunV1Params) (*model.APIRunD
func (c *RunClient) Get(parameters *params.GetRunV1Params) (*model.APIRunDetail,
*workflowapi.Workflow, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -105,9 +120,9 @@ func (c *RunClient) Get(parameters *params.GetRunV1Params) (*model.APIRunDetail,
response, err := c.apiClient.RunService.GetRunV1(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetRunV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, nil, util.NewUserError(err,
@ -130,7 +145,7 @@ func (c *RunClient) Get(parameters *params.GetRunV1Params) (*model.APIRunDetail,
func (c *RunClient) Archive(parameters *params.ArchiveRunV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -139,9 +154,9 @@ func (c *RunClient) Archive(parameters *params.ArchiveRunV1Params) error {
if err != nil {
if defaultError, ok := err.(*params.ListRunsV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -154,7 +169,7 @@ func (c *RunClient) Archive(parameters *params.ArchiveRunV1Params) error {
func (c *RunClient) Unarchive(parameters *params.UnarchiveRunV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -163,9 +178,9 @@ func (c *RunClient) Unarchive(parameters *params.UnarchiveRunV1Params) error {
if err != nil {
if defaultError, ok := err.(*params.ListRunsV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -178,7 +193,7 @@ func (c *RunClient) Unarchive(parameters *params.UnarchiveRunV1Params) error {
func (c *RunClient) Delete(parameters *params.DeleteRunV1Params) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -187,9 +202,9 @@ func (c *RunClient) Delete(parameters *params.DeleteRunV1Params) error {
if err != nil {
if defaultError, ok := err.(*params.ListRunsV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
@ -203,7 +218,7 @@ func (c *RunClient) Delete(parameters *params.DeleteRunV1Params) error {
func (c *RunClient) List(parameters *params.ListRunsV1Params) (
[]*model.APIRun, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -212,9 +227,9 @@ func (c *RunClient) List(parameters *params.ListRunsV1Params) (
if err != nil {
if defaultError, ok := err.(*params.ListRunsV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
@ -256,7 +271,7 @@ func listAllForRun(client RunInterface, parameters *params.ListRunsV1Params, max
}
func (c *RunClient) Terminate(parameters *params.TerminateRunV1Params) error {
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call

14
backend/src/common/client/api_server/run_client_fake.go → backend/src/common/client/api_server/v1/run_client_fake.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server
import (

15
backend/src/common/client/api_server/visualization_client.go → backend/src/common/client/api_server/v1/visualization_client.go
View File

@ -8,6 +8,7 @@ import (
apiclient "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/visualization_client"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/visualization_client/visualization_service"
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/visualization_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -26,7 +27,7 @@ type VisualizationClient struct {
func NewVisualizationClient(clientConfig clientcmd.ClientConfig, debug bool) (
*VisualizationClient, error) {
runtime, err := NewHTTPRuntime(clientConfig, debug)
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating visualization client: %w", err)
}
@ -42,31 +43,31 @@ func NewVisualizationClient(clientConfig clientcmd.ClientConfig, debug bool) (
func NewKubeflowInClusterVisualizationClient(namespace string, debug bool) (
*VisualizationClient, error) {
runtime := NewKubeflowInClusterHTTPRuntime(namespace, debug)
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating upload client
return &VisualizationClient{
apiClient: apiClient,
authInfoWriter: SATokenVolumeProjectionAuth,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *VisualizationClient) Create(parameters *params.CreateVisualizationV1Params) (*model.APIVisualization,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.VisualizationService.CreateVisualizationV1(parameters, PassThroughAuth)
response, err := c.apiClient.VisualizationService.CreateVisualizationV1(parameters, api_server.PassThroughAuth)
if err != nil {
if defaultError, ok := err.(*params.CreateVisualizationV1Default); ok {
err = CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Error, defaultError.Payload.Code)
} else {
err = CreateErrorCouldNotRecoverAPIStatus(err)
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,

0
backend/src/common/client/api_server/visualization_client_fake.go → backend/src/common/client/api_server/v1/visualization_client_fake.go
View File

17
backend/src/common/client/api_server/v2/experiment_client.go
View File

@ -16,7 +16,6 @@ package api_server_v2
import (
"fmt"
"time"
"github.com/go-openapi/strfmt"
apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client"
@ -29,10 +28,6 @@ import (
"k8s.io/client-go/tools/clientcmd"
)
const (
apiServerDefaultTimeout = 35 * time.Second
)
type ExperimentInterface interface {
Create(params *params.CreateExperimentParams) (*model.V2beta1Experiment, error)
Get(params *params.GetExperimentParams) (*model.V2beta1Experiment, error)
@ -78,7 +73,7 @@ func NewKubeflowInClusterExperimentClient(namespace string, debug bool) (
func (c *ExperimentClient) Create(parameters *params.CreateExperimentParams) (*model.V2beta1Experiment,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -96,7 +91,7 @@ func (c *ExperimentClient) Create(parameters *params.CreateExperimentParams) (*m
func (c *ExperimentClient) Get(parameters *params.GetExperimentParams) (*model.V2beta1Experiment,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -114,7 +109,7 @@ func (c *ExperimentClient) Get(parameters *params.GetExperimentParams) (*model.V
func (c *ExperimentClient) List(parameters *params.ListExperimentsParams) (
[]*model.V2beta1Experiment, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -131,7 +126,7 @@ func (c *ExperimentClient) List(parameters *params.ListExperimentsParams) (
func (c *ExperimentClient) Delete(parameters *params.DeleteExperimentParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -178,7 +173,7 @@ func listAllForExperiment(client ExperimentInterface, parameters *params.ListExp
func (c *ExperimentClient) Archive(parameters *params.ArchiveExperimentParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
@ -196,7 +191,7 @@ func (c *ExperimentClient) Archive(parameters *params.ArchiveExperimentParams) e
func (c *ExperimentClient) Unarchive(parameters *params.UnarchiveExperimentParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), apiServerDefaultTimeout)
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call

79
backend/src/common/client/api_server/v2/experiment_client_fake.go
View File

@ -15,20 +15,13 @@
package api_server_v2
import (
"fmt"
"github.com/go-openapi/strfmt"
experimentparams "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
experimentmodel "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_model"
)
const (
v2ExperimentForClientErrorTest = "EXPERIMENT_CLIENT_ERROR"
)
func getDefaultExperiment(id string, name string) *experimentmodel.V2beta1Experiment {
return &experimentmodel.V2beta1Experiment{
func getDefaultExperiment(id string, name string) *model.V2beta1Experiment {
return &model.V2beta1Experiment{
CreatedAt: strfmt.NewDateTime(),
Description: "EXPERIMENT_DESCRIPTION",
ExperimentID: id,
@ -42,63 +35,33 @@ func NewExperimentClientFake() *ExperimentClientFake {
return &ExperimentClientFake{}
}
func (c *ExperimentClientFake) Create(params *experimentparams.CreateExperimentParams) (
*experimentmodel.V2beta1Experiment, error) {
switch params.Body.DisplayName {
case v2ExperimentForClientErrorTest:
return nil, fmt.Errorf(api_server.ClientErrorString)
default:
return getDefaultExperiment("500", params.Body.DisplayName), nil
}
func (c *ExperimentClientFake) Create(parameters *params.CreateExperimentParams) (
*model.V2beta1Experiment, error) {
return getDefaultExperiment("500", parameters.Body.DisplayName), nil
}
func (c *ExperimentClientFake) Get(params *experimentparams.GetExperimentParams) (
*experimentmodel.V2beta1Experiment, error) {
switch params.ExperimentID {
case v2ExperimentForClientErrorTest:
return nil, fmt.Errorf(api_server.ClientErrorString)
default:
return getDefaultExperiment(params.ExperimentID, "EXPERIMENT_NAME"), nil
}
func (c *ExperimentClientFake) Get(parameters *params.GetExperimentParams) (
*model.V2beta1Experiment, error) {
return getDefaultExperiment(parameters.ExperimentID, "EXPERIMENT_NAME"), nil
}
func (c *ExperimentClientFake) List(params *experimentparams.ListExperimentsParams) (
[]*experimentmodel.V2beta1Experiment, int, string, error) {
const (
FirstToken = ""
SecondToken = "SECOND_TOKEN"
FinalToken = ""
)
token := ""
if params.PageToken != nil {
token = *params.PageToken
}
switch token {
case FirstToken:
return []*experimentmodel.V2beta1Experiment{
getDefaultExperiment("100", "MY_FIRST_EXPERIMENT"),
getDefaultExperiment("101", "MY_SECOND_EXPERIMENT"),
}, 2, SecondToken, nil
case SecondToken:
return []*experimentmodel.V2beta1Experiment{
getDefaultExperiment("102", "MY_THIRD_EXPERIMENT"),
}, 1, FinalToken, nil
default:
return nil, 0, "", fmt.Errorf(api_server.InvalidFakeRequest, token)
}
func (c *ExperimentClientFake) List(params *params.ListExperimentsParams) (
[]*model.V2beta1Experiment, int, string, error) {
return []*model.V2beta1Experiment{
getDefaultExperiment("100", "MY_FIRST_EXPERIMENT"),
getDefaultExperiment("101", "MY_SECOND_EXPERIMENT"),
}, 2, "SECOND_TOKEN", nil
}
func (c *ExperimentClientFake) ListAll(params *experimentparams.ListExperimentsParams,
maxResultSize int) ([]*experimentmodel.V2beta1Experiment, error) {
return listAllForExperiment(c, params, maxResultSize)
func (c *ExperimentClientFake) ListAll(params *params.ListExperimentsParams,
maxResultSize int) ([]*model.V2beta1Experiment, error) {
return listAllForExperiment(c, params, 1)
}
func (c *ExperimentClientFake) Archive(params *experimentparams.ArchiveExperimentParams) error {
func (c *ExperimentClientFake) Archive(parameters *params.ArchiveExperimentParams) error {
return nil
}
func (c *ExperimentClientFake) Unarchive(params *experimentparams.UnarchiveExperimentParams) error {
func (c *ExperimentClientFake) Unarchive(parameters *params.UnarchiveExperimentParams) error {
return nil
}

67
backend/src/common/client/api_server/v2/healthz_client.go Normal file
View File

@ -0,0 +1,67 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"fmt"
"github.com/go-openapi/strfmt"
apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/healthz_client"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/healthz_client/healthz_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/healthz_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"k8s.io/client-go/tools/clientcmd"
)
type HealthzInterface interface {
GetHealthz() (*params.GetHealthzOK, error)
}
type HealthzClient struct {
apiClient *apiclient.Healthz
}
func NewHealthzClient(clientConfig clientcmd.ClientConfig, debug bool) (*HealthzClient, error) {
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, err
}
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating upload client
return &HealthzClient{
apiClient: apiClient,
}, nil
}
func (c *HealthzClient) GetHealthz() (*model.V2beta1GetHealthzResponse, error) {
parameters := params.NewGetHealthzParamsWithTimeout(api_server.APIServerDefaultTimeout)
response, err := c.apiClient.HealthzService.GetHealthz(parameters, api_server.PassThroughAuth)
if err != nil {
if defaultError, ok := err.(*params.GetHealthzDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to get Healthz. Params: '%+v'", parameters),
fmt.Sprintf("Failed to get Healthz. Params: '%+v'", parameters))
}
return response.Payload, nil
}

293
backend/src/common/client/api_server/v2/pipeline_client.go Normal file
View File

@ -0,0 +1,293 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"fmt"
"github.com/go-openapi/runtime"
"github.com/go-openapi/strfmt"
apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client/pipeline_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/tools/clientcmd"
)
type PipelineInterface interface {
Create(params *params.CreatePipelineParams) (*model.V2beta1Pipeline, error)
Get(params *params.GetPipelineParams) (*model.V2beta1Pipeline, error)
Delete(params *params.DeletePipelineParams) error
//GetTemplate(params *params.GetTemplateParams) (template.Template, error)
List(params *params.ListPipelinesParams) ([]*model.V2beta1Pipeline, int, string, error)
ListAll(params *params.ListPipelinesParams, maxResultSize int) (
[]*model.V2beta1Pipeline, error)
// UpdateDefaultVersion(params *params.UpdatePipelineDefaultVersionParams) error
}
type PipelineClient struct {
apiClient *apiclient.Pipeline
authInfoWriter runtime.ClientAuthInfoWriter
}
func NewPipelineClient(clientConfig clientcmd.ClientConfig, debug bool) (
*PipelineClient, error) {
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating pipeline client: %w", err)
}
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating pipeline client
return &PipelineClient{
apiClient: apiClient,
}, nil
}
func NewKubeflowInClusterPipelineClient(namespace string, debug bool) (
*PipelineClient, error) {
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating pipeline client
return &PipelineClient{
apiClient: apiClient,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *PipelineClient) Create(parameters *params.CreatePipelineParams) (*model.V2beta1Pipeline,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
parameters.Context = ctx
response, err := c.apiClient.PipelineService.CreatePipeline(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.CreatePipelineDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to create pipeline. Params: '%v'", parameters),
fmt.Sprintf("Failed to create pipeline '%v'", parameters.Body.DisplayName))
}
return response.Payload, nil
}
func (c *PipelineClient) Get(parameters *params.GetPipelineParams) (*model.V2beta1Pipeline,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.PipelineService.GetPipeline(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetPipelineDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to get pipeline. Params: '%v'", parameters),
fmt.Sprintf("Failed to get pipeline '%v'", parameters.PipelineID))
}
return response.Payload, nil
}
func (c *PipelineClient) Delete(parameters *params.DeletePipelineParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.PipelineService.DeletePipeline(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DeletePipelineDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
fmt.Sprintf("Failed to delete pipeline. Params: '%+v'", parameters),
fmt.Sprintf("Failed to delete pipeline '%v'", parameters.PipelineID))
}
return nil
}
func (c *PipelineClient) DeletePipelineVersion(parameters *params.DeletePipelineVersionParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.PipelineService.DeletePipelineVersion(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.DeletePipelineVersionDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
fmt.Sprintf("Failed to delete pipeline version. Params: '%+v'", parameters),
fmt.Sprintf("Failed to delete pipeline version '%v'", parameters.PipelineVersionID))
}
return nil
}
func (c *PipelineClient) List(parameters *params.ListPipelinesParams) (
[]*model.V2beta1Pipeline, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.PipelineService.ListPipelines(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListPipelinesDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
fmt.Sprintf("Failed to list pipelines. Params: '%+v'", parameters),
fmt.Sprintf("Failed to list pipelines"))
}
return response.Payload.Pipelines, int(response.Payload.TotalSize), response.Payload.NextPageToken, nil
}
func (c *PipelineClient) ListAll(parameters *params.ListPipelinesParams, maxResultSize int) (
[]*model.V2beta1Pipeline, error) {
return listAllForPipeline(c, parameters, maxResultSize)
}
func listAllForPipeline(client PipelineInterface, parameters *params.ListPipelinesParams,
maxResultSize int) ([]*model.V2beta1Pipeline, error) {
if maxResultSize < 0 {
maxResultSize = 0
}
allResults := make([]*model.V2beta1Pipeline, 0)
firstCall := true
for (firstCall || (parameters.PageToken != nil && *parameters.PageToken != "")) &&
(len(allResults) < maxResultSize) {
results, _, pageToken, err := client.List(parameters)
if err != nil {
return nil, err
}
allResults = append(allResults, results...)
parameters.PageToken = util.StringPointer(pageToken)
firstCall = false
}
if len(allResults) > maxResultSize {
allResults = allResults[0:maxResultSize]
}
return allResults, nil
}
func (c *PipelineClient) CreatePipelineVersion(parameters *params.CreatePipelineVersionParams) (*model.V2beta1PipelineVersion,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
parameters.Context = ctx
response, err := c.apiClient.PipelineService.CreatePipelineVersion(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.CreatePipelineVersionDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to create pipeline version. Params: '%v'", parameters),
fmt.Sprintf("Failed to create pipeline version for pipeline: '%v'", parameters.PipelineID))
}
return response.Payload, nil
}
func (c *PipelineClient) ListPipelineVersions(parameters *params.ListPipelineVersionsParams) (
[]*model.V2beta1PipelineVersion, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.PipelineService.ListPipelineVersions(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListPipelineVersionsDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
fmt.Sprintf("Failed to list pipeline versions. Params: '%+v'", parameters),
fmt.Sprintf("Failed to list pipeline versions"))
}
return response.Payload.PipelineVersions, int(response.Payload.TotalSize), response.Payload.NextPageToken, nil
}
func (c *PipelineClient) GetPipelineVersion(parameters *params.GetPipelineVersionParams) (*model.V2beta1PipelineVersion,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.PipelineService.GetPipelineVersion(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetPipelineVersionDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to get pipeline version. Params: '%v'", parameters),
fmt.Sprintf("Failed to get pipeline version '%v'", parameters.PipelineVersionID))
}
return response.Payload, nil
}

90
backend/src/common/client/api_server/v2/pipeline_client_fake.go Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"github.com/kubeflow/pipelines/backend/src/apiserver/template"
"google.golang.org/protobuf/types/known/structpb"
workflowapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
"github.com/go-openapi/strfmt"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client/pipeline_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_model"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func getDefaultPipeline(id string) *model.V2beta1Pipeline {
return &model.V2beta1Pipeline{
CreatedAt: strfmt.NewDateTime(),
Description: "PIPELINE_DESCRIPTION",
PipelineID: id,
DisplayName: "PIPELINE_NAME",
}
}
func getDefaultPipelineSpec() *structpb.Struct {
m := make(map[string]interface{})
m["pipelineInfo"] = map[string]interface{}{"name": "MY_NAME"}
spec, _ := structpb.NewStruct(m)
return spec
}
func getDefaultTemplate() template.Template {
tmpl, _ := template.NewArgoTemplateFromWorkflow(&workflowapi.Workflow{
ObjectMeta: metav1.ObjectMeta{
Namespace: "MY_NAMESPACE",
Name: "MY_NAME",
}})
return tmpl
}
func getDefaultWorkflowAsString() string {
tmpl := getDefaultTemplate()
return string(tmpl.Bytes())
}
type PipelineClientFake struct{}
func NewPipelineClientFake() *PipelineClientFake {
return &PipelineClientFake{}
}
func (c *PipelineClientFake) Create(params *params.CreatePipelineParams) (
*model.V2beta1Pipeline, error) {
return getDefaultPipeline(params.Body.PipelineID), nil
}
func (c *PipelineClientFake) Get(params *params.GetPipelineParams) (
*model.V2beta1Pipeline, error) {
return getDefaultPipeline(params.PipelineID), nil
}
func (c *PipelineClientFake) Delete(params *params.DeletePipelineParams) error {
return nil
}
func (c *PipelineClientFake) List(params *params.ListPipelinesParams) (
[]*model.V2beta1Pipeline, int, string, error) {
return []*model.V2beta1Pipeline{
getDefaultPipeline("PIPELINE_ID_100"),
getDefaultPipeline("PIPELINE_ID_101"),
}, 2, "", nil
}
func (c *PipelineClientFake) ListAll(params *params.ListPipelinesParams,
maxResultSize int) ([]*model.V2beta1Pipeline, error) {
return listAllForPipeline(c, params, maxResultSize)
}

151
backend/src/common/client/api_server/v2/pipeline_upload_client.go Normal file
View File

@ -0,0 +1,151 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"context"
"fmt"
"os"
"github.com/go-openapi/runtime"
"github.com/go-openapi/strfmt"
apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/tools/clientcmd"
)
const (
pipelineUploadFieldName = "uploadfile"
pipelineUploadPath = "pipelines/upload"
pipelineUploadServerBasePath = "/api/v2/namespaces/%s/services/ml-pipeline:8888/proxy/apis/v2beta1/%s"
pipelineUploadContentTypeKey = "Content-Type"
pipelineVersionUploadPath = "pipelines/upload_version"
)
type PipelineUploadInterface interface {
UploadFile(filePath string, parameters *params.UploadPipelineParams) (*model.V2beta1Pipeline, error)
}
type PipelineUploadClient struct {
apiClient *apiclient.PipelineUpload
authInfoWriter runtime.ClientAuthInfoWriter
}
func NewPipelineUploadClient(clientConfig clientcmd.ClientConfig, debug bool) (
*PipelineUploadClient, error) {
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating pipeline upload client: %w", err)
}
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating upload client
return &PipelineUploadClient{
apiClient: apiClient,
}, nil
}
func NewKubeflowInClusterPipelineUploadClient(namespace string, debug bool) (
*PipelineUploadClient, error) {
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating upload client
return &PipelineUploadClient{
apiClient: apiClient,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *PipelineUploadClient) UploadFile(filePath string, parameters *params.UploadPipelineParams) (
*model.V2beta1Pipeline, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, util.NewUserErrorWithSingleMessage(err,
fmt.Sprintf("Failed to open file '%s'", filePath))
}
defer file.Close()
parameters.Uploadfile = runtime.NamedReader(filePath, file)
return c.Upload(parameters)
}
func (c *PipelineUploadClient) Upload(parameters *params.UploadPipelineParams) (*model.V2beta1Pipeline,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.PipelineUploadService.UploadPipeline(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.UploadPipelineDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to upload pipeline. Params: '%v'", parameters),
fmt.Sprintf("Failed to upload pipeline"))
}
return response.Payload, nil
}
// UploadPipelineVersion uploads pipeline version from local file.
func (c *PipelineUploadClient) UploadPipelineVersion(filePath string, parameters *params.UploadPipelineVersionParams) (*model.V2beta1PipelineVersion,
error) {
// Get file
file, err := os.Open(filePath)
if err != nil {
return nil, util.NewUserErrorWithSingleMessage(err,
fmt.Sprintf("Failed to open file '%s'", filePath))
}
defer file.Close()
parameters.Uploadfile = runtime.NamedReader(filePath, file)
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.PipelineUploadService.UploadPipelineVersion(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.UploadPipelineVersionDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to upload pipeline version. Params: '%v'", parameters),
fmt.Sprintf("Failed to upload pipeline version"))
}
return response.Payload, nil
}

56
backend/src/common/client/api_server/v2/pipeline_upload_client_fake.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"fmt"
"github.com/go-openapi/strfmt"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_model"
)
const (
FileForDefaultTest = "./samples/parameters.yaml"
FileForClientErrorTest = "./samples/hello-world.yaml"
ClientErrorString = "Error with client"
InvalidFakeRequest = "Invalid fake request, don't know how to handle '%s' in the fake client."
)
func getDefaultUploadedPipeline() *model.V2beta1Pipeline {
return &model.V2beta1Pipeline{
PipelineID: "500",
CreatedAt: strfmt.NewDateTime(),
DisplayName: "PIPELINE_NAME",
Description: "PIPELINE_DESCRIPTION",
}
}
type PipelineUploadClientFake struct{}
func NewPipelineUploadClientFake() *PipelineUploadClientFake {
return &PipelineUploadClientFake{}
}
func (c *PipelineUploadClientFake) UploadFile(filePath string,
parameters *params.UploadPipelineParams) (*model.V2beta1Pipeline, error) {
switch filePath {
case FileForClientErrorTest:
return nil, fmt.Errorf(ClientErrorString)
default:
return getDefaultUploadedPipeline(), nil
}
}

210
backend/src/common/client/api_server/v2/recurring_run_client.go Normal file
View File

@ -0,0 +1,210 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"fmt"
"github.com/go-openapi/runtime"
"github.com/go-openapi/strfmt"
apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_client"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_client/recurring_run_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/tools/clientcmd"
)
type RecurringRunInterface interface {
Create(params *params.CreateRecurringRunParams) (*model.V2beta1RecurringRun, error)
Get(params *params.GetRecurringRunParams) (*model.V2beta1RecurringRun, error)
Delete(params *params.DeleteRecurringRunParams) error
Enable(params *params.EnableRecurringRunParams) error
Disable(params *params.DisableRecurringRunParams) error
List(params *params.ListRecurringRunsParams) ([]*model.V2beta1RecurringRun, int, string, error)
ListAll(params *params.ListRecurringRunsParams, maxResultSize int) ([]*model.V2beta1RecurringRun, error)
}
type RecurringRunClient struct {
apiClient *apiclient.RecurringRun
authInfoWriter runtime.ClientAuthInfoWriter
}
func NewRecurringRunClient(clientConfig clientcmd.ClientConfig, debug bool) (
*RecurringRunClient, error) {
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating job client: %w", err)
}
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating job client
return &RecurringRunClient{
apiClient: apiClient,
}, nil
}
func NewKubeflowInClusterRecurringRunClient(namespace string, debug bool) (
*RecurringRunClient, error) {
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating job client
return &RecurringRunClient{
apiClient: apiClient,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *RecurringRunClient) Create(parameters *params.CreateRecurringRunParams) (*model.V2beta1RecurringRun,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.RecurringRunService.CreateRecurringRun(parameters)
if err != nil {
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to create job. Params: '%+v'. Body: '%+v'", parameters, parameters.Body),
fmt.Sprintf("Failed to create job '%v'", parameters.Body.DisplayName))
}
return response.Payload, nil
}
func (c *RecurringRunClient) Get(parameters *params.GetRecurringRunParams) (*model.V2beta1RecurringRun,
error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.RecurringRunService.GetRecurringRun(parameters)
if err != nil {
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to get job. Params: '%+v'", parameters),
fmt.Sprintf("Failed to get job '%v'", parameters.RecurringRunID))
}
return response.Payload, nil
}
func (c *RecurringRunClient) Delete(parameters *params.DeleteRecurringRunParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RecurringRunService.DeleteRecurringRun(parameters)
if err != nil {
return util.NewUserError(err,
fmt.Sprintf("Failed to get job. Params: '%+v'", parameters),
fmt.Sprintf("Failed to get job '%v'", parameters.RecurringRunID))
}
return nil
}
func (c *RecurringRunClient) Enable(parameters *params.EnableRecurringRunParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RecurringRunService.EnableRecurringRun(parameters)
if err != nil {
return util.NewUserError(err,
fmt.Sprintf("Failed to enable job. Params: '%+v'", parameters),
fmt.Sprintf("Failed to enable job '%v'", parameters.RecurringRunID))
}
return nil
}
func (c *RecurringRunClient) Disable(parameters *params.DisableRecurringRunParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RecurringRunService.DisableRecurringRun(parameters)
if err != nil {
return util.NewUserError(err,
fmt.Sprintf("Failed to disable job. Params: '%+v'", parameters),
fmt.Sprintf("Failed to disable job '%v'", parameters.RecurringRunID))
}
return nil
}
func (c *RecurringRunClient) List(parameters *params.ListRecurringRunsParams) (
[]*model.V2beta1RecurringRun, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.RecurringRunService.ListRecurringRuns(parameters)
if err != nil {
return nil, 0, "", util.NewUserError(err,
fmt.Sprintf("Failed to list jobs. Params: '%+v'", parameters),
fmt.Sprintf("Failed to list jobs"))
}
return response.Payload.RecurringRuns, int(response.Payload.TotalSize), response.Payload.NextPageToken, nil
}
func (c *RecurringRunClient) ListAll(parameters *params.ListRecurringRunsParams, maxResultSize int) (
[]*model.V2beta1RecurringRun, error) {
return listAllForJob(c, parameters, maxResultSize)
}
func listAllForJob(client RecurringRunInterface, parameters *params.ListRecurringRunsParams,
maxResultSize int) ([]*model.V2beta1RecurringRun, error) {
if maxResultSize < 0 {
maxResultSize = 0
}
allResults := make([]*model.V2beta1RecurringRun, 0)
firstCall := true
for (firstCall || (parameters.PageToken != nil && *parameters.PageToken != "")) &&
(len(allResults) < maxResultSize) {
results, _, pageToken, err := client.List(parameters)
if err != nil {
return nil, err
}
allResults = append(allResults, results...)
parameters.PageToken = util.StringPointer(pageToken)
firstCall = false
}
if len(allResults) > maxResultSize {
allResults = allResults[0:maxResultSize]
}
return allResults, nil
}

71
backend/src/common/client/api_server/v2/recurring_run_client_fake.go Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"github.com/go-openapi/strfmt"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_client/recurring_run_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_model"
)
func getDefaultJob(id string, name string) *model.V2beta1RecurringRun {
return &model.V2beta1RecurringRun{
CreatedAt: strfmt.NewDateTime(),
Description: "RECURRING_RUN_DESCRIPTION",
RecurringRunID: id,
DisplayName: name,
}
}
type RecurringRunClientFake struct{}
func NewRecurringRunClientFake() *RecurringRunClientFake {
return &RecurringRunClientFake{}
}
func (c *RecurringRunClientFake) Create(params *params.CreateRecurringRunParams) (
*model.V2beta1RecurringRun, error) {
return getDefaultJob("500", params.Body.DisplayName), nil
}
func (c *RecurringRunClientFake) Get(params *params.GetRecurringRunParams) (
*model.V2beta1RecurringRun, error) {
return getDefaultJob(params.RecurringRunID, "RECURRING_RUN_NAME"), nil
}
func (c *RecurringRunClientFake) Delete(params *params.DeleteRecurringRunParams) error {
return nil
}
func (c *RecurringRunClientFake) Enable(params *params.EnableRecurringRunParams) error {
return nil
}
func (c *RecurringRunClientFake) Disable(params *params.DisableRecurringRunParams) error {
return nil
}
func (c *RecurringRunClientFake) List(params *params.ListRecurringRunsParams) (
[]*model.V2beta1RecurringRun, int, string, error) {
return []*model.V2beta1RecurringRun{
getDefaultJob("100", "MY_FIRST_RECURRING_RUN"),
getDefaultJob("101", "MY_SECOND_RECURRING_RUN"),
}, 2, "", nil
}
func (c *RecurringRunClientFake) ListAll(params *params.ListRecurringRunsParams,
maxResultSize int) ([]*model.V2beta1RecurringRun, error) {
return listAllForJob(c, params, maxResultSize)
}

263
backend/src/common/client/api_server/v2/run_client.go Normal file
View File

@ -0,0 +1,263 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"fmt"
"github.com/go-openapi/runtime"
"github.com/go-openapi/strfmt"
apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/kubeflow/pipelines/backend/src/common/util"
"golang.org/x/net/context"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/tools/clientcmd"
)
type RunInterface interface {
Archive(params *params.ArchiveRunParams) error
Create(params *params.CreateRunParams) (*model.V2beta1Run, error)
Get(params *params.GetRunParams) (*model.V2beta1Run, error)
List(params *params.ListRunsParams) ([]*model.V2beta1Run, int, string, error)
ListAll(params *params.ListRunsParams, maxResultSize int) ([]*model.V2beta1Run, error)
Unarchive(params *params.UnarchiveRunParams) error
Terminate(params *params.TerminateRunParams) error
}
type RunClient struct {
apiClient *apiclient.Run
authInfoWriter runtime.ClientAuthInfoWriter
}
func NewRunClient(clientConfig clientcmd.ClientConfig, debug bool) (
*RunClient, error) {
runtime, err := api_server.NewHTTPRuntime(clientConfig, debug)
if err != nil {
return nil, fmt.Errorf("Error occurred when creating run client: %w", err)
}
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating run client
return &RunClient{
apiClient: apiClient,
}, nil
}
func NewKubeflowInClusterRunClient(namespace string, debug bool) (
*RunClient, error) {
runtime := api_server.NewKubeflowInClusterHTTPRuntime(namespace, debug)
apiClient := apiclient.New(runtime, strfmt.Default)
// Creating run client
return &RunClient{
apiClient: apiClient,
authInfoWriter: api_server.SATokenVolumeProjectionAuth,
}, nil
}
func (c *RunClient) Create(parameters *params.CreateRunParams) (*model.V2beta1Run, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.RunService.CreateRun(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetRunDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to create run. Params: '%+v'", parameters),
fmt.Sprintf("Failed to create run '%v'", parameters.Body.DisplayName))
}
return response.Payload, nil
}
func (c *RunClient) Get(parameters *params.GetRunParams) (*model.V2beta1Run, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.RunService.GetRun(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.GetRunDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, util.NewUserError(err,
fmt.Sprintf("Failed to get run. Params: '%+v'", parameters),
fmt.Sprintf("Failed to get run '%v'", parameters.RunID))
}
return response.Payload, nil
}
func (c *RunClient) Archive(parameters *params.ArchiveRunParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RunService.ArchiveRun(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListRunsDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
fmt.Sprintf("Failed to archive runs. Params: '%+v'", parameters),
fmt.Sprintf("Failed to archive runs"))
}
return nil
}
func (c *RunClient) Unarchive(parameters *params.UnarchiveRunParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RunService.UnarchiveRun(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListRunsDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
fmt.Sprintf("Failed to unarchive runs. Params: '%+v'", parameters),
fmt.Sprintf("Failed to unarchive runs"))
}
return nil
}
func (c *RunClient) Delete(parameters *params.DeleteRunParams) error {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RunService.DeleteRun(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListRunsDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return util.NewUserError(err,
fmt.Sprintf("Failed to delete runs. Params: '%+v'", parameters),
fmt.Sprintf("Failed to delete runs"))
}
return nil
}
func (c *RunClient) List(parameters *params.ListRunsParams) (
[]*model.V2beta1Run, int, string, error) {
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
response, err := c.apiClient.RunService.ListRuns(parameters, c.authInfoWriter)
if err != nil {
if defaultError, ok := err.(*params.ListRunsDefault); ok {
err = api_server.CreateErrorFromAPIStatus(defaultError.Payload.Message, defaultError.Payload.Code)
} else {
err = api_server.CreateErrorCouldNotRecoverAPIStatus(err)
}
return nil, 0, "", util.NewUserError(err,
fmt.Sprintf("Failed to list runs. Params: '%+v'", parameters),
fmt.Sprintf("Failed to list runs"))
}
return response.Payload.Runs, int(response.Payload.TotalSize), response.Payload.NextPageToken, nil
}
func (c *RunClient) ListAll(parameters *params.ListRunsParams, maxResultSize int) (
[]*model.V2beta1Run, error) {
return listAllForRun(c, parameters, maxResultSize)
}
func listAllForRun(client RunInterface, parameters *params.ListRunsParams, maxResultSize int) (
[]*model.V2beta1Run, error) {
if maxResultSize < 0 {
maxResultSize = 0
}
allResults := make([]*model.V2beta1Run, 0)
firstCall := true
for (firstCall || (parameters.PageToken != nil && *parameters.PageToken != "")) &&
(len(allResults) < maxResultSize) {
results, _, pageToken, err := client.List(parameters)
if err != nil {
return nil, err
}
allResults = append(allResults, results...)
parameters.PageToken = util.StringPointer(pageToken)
firstCall = false
}
if len(allResults) > maxResultSize {
allResults = allResults[0:maxResultSize]
}
return allResults, nil
}
func (c *RunClient) Terminate(parameters *params.TerminateRunParams) error {
ctx, cancel := context.WithTimeout(context.Background(), api_server.APIServerDefaultTimeout)
defer cancel()
// Make service call
parameters.Context = ctx
_, err := c.apiClient.RunService.TerminateRun(parameters, c.authInfoWriter)
if err != nil {
return util.NewUserError(err,
fmt.Sprintf("Failed to terminate run. Params: %+v", parameters),
fmt.Sprintf("Failed to terminate run %v", parameters.RunID))
}
return nil
}

71
backend/src/common/client/api_server/v2/run_client_fake.go Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api_server_v2
import (
"fmt"
"github.com/go-openapi/strfmt"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_model"
)
func getDefaultRun(id string, name string) *model.V2beta1Run {
return &model.V2beta1Run{
CreatedAt: strfmt.NewDateTime(),
RunID: id,
DisplayName: name,
}
}
type RunClientFake struct{}
func NewRunClientFake() *RunClientFake {
return &RunClientFake{}
}
func (c *RunClientFake) Create(params *params.CreateRunParams) (*model.V2beta1Run, error) {
return getDefaultRun("100", "RUN_NAME"), nil
}
func (c *RunClientFake) Get(params *params.GetRunParams) (*model.V2beta1Run, error) {
return getDefaultRun(params.RunID, "RUN_NAME"), nil
}
func (c *RunClientFake) List(params *params.ListRunsParams) (
[]*model.V2beta1Run, int, string, error) {
return []*model.V2beta1Run{
getDefaultRun("100", "MY_FIRST_RUN"),
getDefaultRun("101", "MY_SECOND_RUN"),
}, 2, "", nil
}
func (c *RunClientFake) ListAll(params *params.ListRunsParams, maxResultSize int) (
[]*model.V2beta1Run, error) {
return listAllForRun(c, params, maxResultSize)
}
func (c *RunClientFake) Archive(params *params.ArchiveRunParams) error {
return nil
}
func (c *RunClientFake) Unarchive(params *params.UnarchiveRunParams) error {
return nil
}
func (c *RunClientFake) Terminate(params *params.TerminateRunParams) error {
return fmt.Errorf(InvalidFakeRequest, params.RunID)
}

2
backend/test/initialization/initialization_test.go
View File

@ -19,7 +19,7 @@ import (
"github.com/golang/glog"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/experiment_client/experiment_service"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"

16
backend/test/integration/experiment_api_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -12,7 +26,7 @@ import (
uploadParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
runParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"

14
backend/test/integration/flags.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (

16
backend/test/integration/healthz_api_test.go
View File

@ -1,10 +1,24 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"testing"
"github.com/golang/glog"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"

16
backend/test/integration/job_api_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -20,7 +34,7 @@ import (
runParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/apiserver/client"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"

16
backend/test/integration/pipeline_api_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -10,7 +24,7 @@ import (
model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_model"
uploadParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
pipelinetemplate "github.com/kubeflow/pipelines/backend/src/apiserver/template"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"

16
backend/test/integration/pipeline_version_api_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -10,7 +24,7 @@ import (
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_model"
uploadParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
pipelinetemplate "github.com/kubeflow/pipelines/backend/src/apiserver/template"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"

16
backend/test/integration/run_api_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -12,7 +26,7 @@ import (
uploadParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
runparams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"

13
backend/test/integration/run_tests_locally.sh
View File

@ -1,4 +1,17 @@
#!/bin/bash
# Copyright 2018-2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e

16
backend/test/integration/upgrade_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -17,7 +31,7 @@ import (
runParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_model"
pipelinetemplate "github.com/kubeflow/pipelines/backend/src/apiserver/template"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"

16
backend/test/integration/visualization_api_test.go
View File

@ -1,3 +1,17 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
@ -6,7 +20,7 @@ import (
"github.com/golang/glog"
params "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/visualization_client/visualization_service"
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/visualization_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"

2
backend/test/test_utils.go
View File

@ -31,7 +31,7 @@ import (
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_model"
runparams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
"github.com/kubeflow/pipelines/backend/src/common/util"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"

11
backend/test/v2/initialization/README.md Normal file
View File

@ -0,0 +1,11 @@
## Api Server Integration Tests
### WARNING
**These integration tests will delete all the data in your KFP instance, please only use a test cluster to run these.**
### How to run
1. Configure kubectl to connect to your kfp cluster.
2. Run the following for all integration tests: `NAMESPACE=<kfp-namespace> ./run_tests_locally.sh`.
3. Or run the following to select certain tests: `NAMESPACE=<kfp-namespace> ./run_tests_locally.sh -testify.m Job`.
Reference: https://stackoverflow.com/a/43312451

390
backend/test/v2/integration/experiment_api_test.go Normal file
View File

@ -0,0 +1,390 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"testing"
"time"
"github.com/golang/glog"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_model"
upload_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
recurring_run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_client/recurring_run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_model"
run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_model"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
"github.com/kubeflow/pipelines/backend/src/common/util"
test "github.com/kubeflow/pipelines/backend/test/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type ExperimentApiTest struct {
suite.Suite
namespace string
resourceNamespace string
experimentClient *api_server.ExperimentClient
pipelineClient *api_server.PipelineClient
pipelineUploadClient *api_server.PipelineUploadClient
runClient *api_server.RunClient
recurringRunClient *api_server.RecurringRunClient
}
// Check the namespace have ML job installed and ready
func (s *ExperimentApiTest) SetupTest() {
if !*runIntegrationTests {
s.T().SkipNow()
return
}
if !*isDevMode {
err := test.WaitForReady(*namespace, *initializeTimeout)
if err != nil {
glog.Exitf("Failed to initialize test. Error: %v", err)
}
}
s.namespace = *namespace
var newExperimentClient func() (*api_server.ExperimentClient, error)
var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
var newPipelineClient func() (*api_server.PipelineClient, error)
var newRunClient func() (*api_server.RunClient, error)
var newRecurringRunClient func() (*api_server.RecurringRunClient, error)
if *isKubeflowMode {
s.resourceNamespace = *resourceNamespace
newExperimentClient = func() (*api_server.ExperimentClient, error) {
return api_server.NewKubeflowInClusterExperimentClient(s.namespace, *isDebugMode)
}
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewKubeflowInClusterPipelineUploadClient(s.namespace, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewKubeflowInClusterPipelineClient(s.namespace, *isDebugMode)
}
newRunClient = func() (*api_server.RunClient, error) {
return api_server.NewKubeflowInClusterRunClient(s.namespace, *isDebugMode)
}
newRecurringRunClient = func() (*api_server.RecurringRunClient, error) {
return api_server.NewKubeflowInClusterRecurringRunClient(s.namespace, *isDebugMode)
}
} else {
clientConfig := test.GetClientConfig(*namespace)
newExperimentClient = func() (*api_server.ExperimentClient, error) {
return api_server.NewExperimentClient(clientConfig, *isDebugMode)
}
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewPipelineUploadClient(clientConfig, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewPipelineClient(clientConfig, *isDebugMode)
}
newRunClient = func() (*api_server.RunClient, error) {
return api_server.NewRunClient(clientConfig, *isDebugMode)
}
newRecurringRunClient = func() (*api_server.RecurringRunClient, error) {
return api_server.NewRecurringRunClient(clientConfig, *isDebugMode)
}
}
var err error
s.experimentClient, err = newExperimentClient()
if err != nil {
glog.Exitf("Failed to get experiment client. Error: %v", err)
}
s.pipelineUploadClient, err = newPipelineUploadClient()
if err != nil {
glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
}
s.pipelineClient, err = newPipelineClient()
if err != nil {
glog.Exitf("Failed to get pipeline client. Error: %s", err.Error())
}
s.runClient, err = newRunClient()
if err != nil {
glog.Exitf("Failed to get run client. Error: %s", err.Error())
}
s.recurringRunClient, err = newRecurringRunClient()
if err != nil {
glog.Exitf("Failed to get job client. Error: %s", err.Error())
}
s.cleanUp()
}
func (s *ExperimentApiTest) TestExperimentAPI() {
t := s.T()
/* ---------- Verify only default experiment exists ---------- */
experiments, totalSize, _, err := test.ListAllExperiment(s.experimentClient, s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, totalSize)
assert.True(t, len(experiments) == 1)
/* ---------- Create a new experiment ---------- */
experiment := test.MakeExperiment("training", "my first experiment", s.resourceNamespace)
expectedTrainingExperiment := test.MakeExperiment("training", "my first experiment", s.resourceNamespace)
trainingExperiment, err := s.experimentClient.Create(&params.CreateExperimentParams{
Body: experiment,
})
assert.Nil(t, err)
expectedTrainingExperiment.ExperimentID = trainingExperiment.ExperimentID
expectedTrainingExperiment.CreatedAt = trainingExperiment.CreatedAt
expectedTrainingExperiment.StorageState = "STORAGESTATE_AVAILABLE"
expectedTrainingExperiment.Namespace = trainingExperiment.Namespace
assert.Equal(t, expectedTrainingExperiment, trainingExperiment)
/* ---------- Create an experiment with same name. Should fail due to name uniqueness ---------- */
_, err = s.experimentClient.Create(&params.CreateExperimentParams{Body: experiment})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Please specify a new name")
/* ---------- Create a few more new experiment ---------- */
// 1 second interval. This ensures they can be sorted by create time in expected order.
time.Sleep(1 * time.Second)
experiment = test.MakeExperiment("prediction", "my second experiment", s.resourceNamespace)
_, err = s.experimentClient.Create(&params.CreateExperimentParams{
Body: experiment,
})
time.Sleep(1 * time.Second)
experiment = test.MakeExperiment("moonshot", "my second experiment", s.resourceNamespace)
_, err = s.experimentClient.Create(&params.CreateExperimentParams{
Body: experiment,
})
assert.Nil(t, err)
/* ---------- Verify list experiments works ---------- */
experiments, totalSize, nextPageToken, err := test.ListAllExperiment(s.experimentClient, s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 4, len(experiments))
for _, e := range experiments {
// Sampling one of the experiments and verify the result is expected.
if e.DisplayName == "training" {
assert.Equal(t, expectedTrainingExperiment, trainingExperiment)
}
}
/* ---------- Verify list experiments sorted by names ---------- */
experiments, totalSize, nextPageToken, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("name"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 2, len(experiments))
assert.Equal(t, "Default", experiments[0].DisplayName)
assert.Equal(t, "moonshot", experiments[1].DisplayName)
assert.NotEmpty(t, nextPageToken)
experiments, totalSize, nextPageToken, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageToken: util.StringPointer(nextPageToken),
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("name"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 2, len(experiments))
assert.Equal(t, "prediction", experiments[0].DisplayName)
assert.Equal(t, "training", experiments[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- Verify list experiments sorted by creation time ---------- */
experiments, totalSize, nextPageToken, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("created_at"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 2, len(experiments))
assert.Equal(t, "Default", experiments[0].DisplayName)
assert.Equal(t, "training", experiments[1].DisplayName)
assert.NotEmpty(t, nextPageToken)
experiments, totalSize, nextPageToken, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageToken: util.StringPointer(nextPageToken),
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("created_at"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 2, len(experiments))
assert.Equal(t, "prediction", experiments[0].DisplayName)
assert.Equal(t, "moonshot", experiments[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- List experiments sort by unsupported field. Should fail. ---------- */
_, _, _, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("unknownfield"),
},
s.resourceNamespace)
assert.NotNil(t, err)
/* ---------- List experiments sorted by names descend order ---------- */
experiments, totalSize, nextPageToken, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("name desc"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 2, len(experiments))
assert.Equal(t, "training", experiments[0].DisplayName)
assert.Equal(t, "prediction", experiments[1].DisplayName)
assert.NotEmpty(t, nextPageToken)
experiments, totalSize, nextPageToken, err = test.ListExperiment(
s.experimentClient,
&params.ListExperimentsParams{
PageToken: util.StringPointer(nextPageToken),
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("name desc"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 4, totalSize)
assert.Equal(t, 2, len(experiments))
assert.Equal(t, "moonshot", experiments[0].DisplayName)
assert.Equal(t, "Default", experiments[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- Verify get experiment works ---------- */
experiment, err = s.experimentClient.Get(&params.GetExperimentParams{ExperimentID: trainingExperiment.ExperimentID})
assert.Nil(t, err)
assert.Equal(t, expectedTrainingExperiment, experiment)
/* ---------- Create a pipeline version and two runs and two jobs -------------- */
pipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", upload_params.NewUploadPipelineParams())
assert.Nil(t, err)
time.Sleep(1 * time.Second)
pipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion(
"../resources/hello-world.yaml", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("hello-world-version"),
Pipelineid: util.StringPointer(pipeline.PipelineID),
})
assert.Nil(t, err)
createRunRequest := &run_params.CreateRunParams{Body: &run_model.V2beta1Run{
DisplayName: "hello world",
Description: "this is hello world",
ExperimentID: experiment.ExperimentID,
PipelineVersionReference: &run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineVersion.PipelineID,
PipelineVersionID: pipelineVersion.PipelineVersionID,
},
}}
run1, err := s.runClient.Create(createRunRequest)
assert.Nil(t, err)
run2, err := s.runClient.Create(createRunRequest)
assert.Nil(t, err)
/* ---------- Create a new hello world job by specifying pipeline ID ---------- */
createRecurringRunRequest := &recurring_run_params.CreateRecurringRunParams{Body: &recurring_run_model.V2beta1RecurringRun{
DisplayName: "hello world",
Description: "this is hello world",
ExperimentID: experiment.ExperimentID,
PipelineVersionReference: &recurring_run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineVersion.PipelineID,
PipelineVersionID: pipelineVersion.PipelineVersionID,
},
MaxConcurrency: 10,
Status: recurring_run_model.V2beta1RecurringRunStatusENABLED,
}}
recurringRun1, err := s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
recurringRun2, err := s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
/* ---------- Archive an experiment -----------------*/
err = s.experimentClient.Archive(&params.ArchiveExperimentParams{ExperimentID: trainingExperiment.ExperimentID})
/* ---------- Verify experiment and its runs ------- */
experiment, err = s.experimentClient.Get(&params.GetExperimentParams{ExperimentID: trainingExperiment.ExperimentID})
assert.Nil(t, err)
assert.Equal(t, experiment_model.V2beta1ExperimentStorageStateARCHIVED, experiment.StorageState)
retrievedRun1, err := s.runClient.Get(&run_params.GetRunParams{RunID: run1.RunID})
assert.Nil(t, err)
assert.Equal(t, run_model.V2beta1RunStorageStateARCHIVED, retrievedRun1.StorageState)
retrievedRun2, err := s.runClient.Get(&run_params.GetRunParams{RunID: run2.RunID})
assert.Nil(t, err)
assert.Equal(t, run_model.V2beta1RunStorageStateARCHIVED, retrievedRun2.StorageState)
retrievedRecurringRun1, err := s.recurringRunClient.Get(&recurring_run_params.GetRecurringRunParams{RecurringRunID: recurringRun1.RecurringRunID})
assert.Nil(t, err)
assert.Equal(t, recurring_run_model.V2beta1RecurringRunStatusDISABLED, retrievedRecurringRun1.Status)
retrievedRecurringRun2, err := s.recurringRunClient.Get(&recurring_run_params.GetRecurringRunParams{RecurringRunID: recurringRun2.RecurringRunID})
assert.Nil(t, err)
assert.Equal(t, recurring_run_model.V2beta1RecurringRunStatusDISABLED, retrievedRecurringRun2.Status)
/* ---------- Unarchive an experiment -----------------*/
err = s.experimentClient.Unarchive(&params.UnarchiveExperimentParams{ExperimentID: trainingExperiment.ExperimentID})
/* ---------- Verify experiment and its runs and jobs --------- */
experiment, err = s.experimentClient.Get(&params.GetExperimentParams{ExperimentID: trainingExperiment.ExperimentID})
assert.Nil(t, err)
assert.Equal(t, experiment_model.V2beta1ExperimentStorageStateAVAILABLE, experiment.StorageState)
retrievedRun1, err = s.runClient.Get(&run_params.GetRunParams{RunID: run1.RunID})
assert.Nil(t, err)
assert.Equal(t, run_model.V2beta1RunStorageStateARCHIVED, retrievedRun1.StorageState)
retrievedRun2, err = s.runClient.Get(&run_params.GetRunParams{RunID: run2.RunID})
assert.Nil(t, err)
assert.Equal(t, run_model.V2beta1RunStorageStateARCHIVED, retrievedRun2.StorageState)
retrievedRecurringRun1, err = s.recurringRunClient.Get(&recurring_run_params.GetRecurringRunParams{RecurringRunID: recurringRun1.RecurringRunID})
assert.Nil(t, err)
assert.Equal(t, recurring_run_model.V2beta1RecurringRunStatusDISABLED, retrievedRecurringRun1.Status)
retrievedRecurringRun2, err = s.recurringRunClient.Get(&recurring_run_params.GetRecurringRunParams{RecurringRunID: recurringRun2.RecurringRunID})
assert.Nil(t, err)
assert.Equal(t, recurring_run_model.V2beta1RecurringRunStatusDISABLED, retrievedRecurringRun2.Status)
}
func V2TestExperimentAPI(t *testing.T) {
suite.Run(t, new(ExperimentApiTest))
}
func (s *ExperimentApiTest) TearDownSuite() {
if *runIntegrationTests {
if !*isDevMode {
s.cleanUp()
}
}
}
func (s *ExperimentApiTest) cleanUp() {
test.DeleteAllRuns(s.runClient, s.resourceNamespace, s.T())
test.DeleteAllRecurringRuns(s.recurringRunClient, s.resourceNamespace, s.T())
test.DeleteAllPipelines(s.pipelineClient, s.T())
test.DeleteAllExperiments(s.experimentClient, s.resourceNamespace, s.T())
}

41
backend/test/v2/integration/flags.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"flag"
"time"
)
var (
namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
runIntegrationTests = flag.Bool("runIntegrationTests", false, "Whether to also run integration tests that call the service")
runUpgradeTests = flag.Bool("runUpgradeTests", false, "Whether to run upgrade tests")
)
/**
* Differences in dev mode:
* 1. Resources are not cleaned up when a test finishes, so that developer can debug manually.
* 2. One step that doesn't work locally is skipped.
*/
var isDevMode = flag.Bool("isDevMode", false, "Dev mode helps local development of integration tests")
var isDebugMode = flag.Bool("isDebugMode", false, "Whether to enable debug mode. Debug mode will log more diagnostics messages.")
var (
isKubeflowMode = flag.Bool("isKubeflowMode", false, "Runs tests in full Kubeflow mode")
resourceNamespace = flag.String("resourceNamespace", "", "The namespace that will store the test resources in Kubeflow mode")
)

78
backend/test/v2/integration/healthz_api_test.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"testing"
"github.com/golang/glog"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
test "github.com/kubeflow/pipelines/backend/test/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type HealthzApiTest struct {
suite.Suite
namespace string
healthzClient *api_server.HealthzClient
}
// Check the namespace have ML job installed and ready
func (s *HealthzApiTest) SetupTest() {
if !*runIntegrationTests {
s.T().SkipNow()
return
}
if !*isDevMode {
err := test.WaitForReady(*namespace, *initializeTimeout)
if err != nil {
glog.Exitf("Failed to initialize test. Error: %v", err)
}
}
s.namespace = *namespace
clientConfig := test.GetClientConfig(*namespace)
var err error
s.healthzClient, err = api_server.NewHealthzClient(clientConfig, false)
if err != nil {
glog.Exitf("Failed to get healthz client. Error: %v", err)
}
s.cleanUp()
}
func (s *HealthzApiTest) TearDownSuite() {
if *runIntegrationTests {
if !*isDevMode {
s.cleanUp()
}
}
}
func (s *HealthzApiTest) cleanUp() {
}
func (s *HealthzApiTest) TestHealthzAPI() {
t := s.T()
/* ---------- Verify healthz response ---------- */
healthzResp, err := s.healthzClient.GetHealthz()
assert.Nil(t, err)
assert.NotNil(t, healthzResp)
}
func TestHealthzAPI(t *testing.T) {
suite.Run(t, new(HealthzApiTest))
}

278
backend/test/v2/integration/pipeline_api_test.go Normal file
View File

@ -0,0 +1,278 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"testing"
"time"
"github.com/golang/glog"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client/pipeline_service"
model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_model"
upload_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
"github.com/kubeflow/pipelines/backend/src/common/util"
test "github.com/kubeflow/pipelines/backend/test/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// This test suit tests various methods to import pipeline to pipeline system, including
// - upload v2 pipeline spec JSON file
// - upload yaml file
// - upload tarball file
// - providing YAML file url
// - providing tarball file url
type PipelineApiTest struct {
suite.Suite
namespace string
resourceNamespace string
pipelineClient *api_server.PipelineClient
pipelineUploadClient *api_server.PipelineUploadClient
}
// Check the namespace have ML job installed and ready
func (s *PipelineApiTest) SetupTest() {
if !*runIntegrationTests {
s.T().SkipNow()
return
}
if !*isDevMode {
err := test.WaitForReady(*namespace, *initializeTimeout)
if err != nil {
glog.Exitf("Failed to initialize test. Error: %s", err.Error())
}
}
s.namespace = *namespace
var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
var newPipelineClient func() (*api_server.PipelineClient, error)
if *isKubeflowMode {
s.resourceNamespace = *resourceNamespace
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewKubeflowInClusterPipelineUploadClient(s.namespace, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewKubeflowInClusterPipelineClient(s.namespace, *isDebugMode)
}
} else {
clientConfig := test.GetClientConfig(*namespace)
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewPipelineUploadClient(clientConfig, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewPipelineClient(clientConfig, *isDebugMode)
}
}
var err error
s.pipelineUploadClient, err = newPipelineUploadClient()
if err != nil {
glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
}
s.pipelineClient, err = newPipelineClient()
if err != nil {
glog.Exitf("Failed to get pipeline client. Error: %s", err.Error())
}
s.cleanUp()
}
func (s *PipelineApiTest) TestPipelineAPI() {
t := s.T()
test.DeleteAllPipelines(s.pipelineClient, t)
/* ------ Upload v2 pipeline spec YAML --------*/
helloPipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", upload_params.NewUploadPipelineParams())
require.Nil(t, err)
assert.Equal(t, "hello-world.yaml", helloPipeline.DisplayName)
/* ---------- Upload pipelines YAML ---------- */
time.Sleep(1 * time.Second)
argumentYAMLPipeline, err := s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", upload_params.NewUploadPipelineParams())
require.Nil(t, err)
assert.Equal(t, "arguments-parameters.yaml", argumentYAMLPipeline.DisplayName)
/* ---------- Upload the same pipeline again. Should fail due to name uniqueness ---------- */
_, err = s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", upload_params.NewUploadPipelineParams())
require.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to upload pipeline")
/* ---------- Import pipeline YAML by URL ---------- */
time.Sleep(1 * time.Second)
sequentialPipeline, err := s.pipelineClient.Create(&params.CreatePipelineParams{
Body: &model.V2beta1Pipeline{DisplayName: "sequential"},
})
require.Nil(t, err)
assert.Equal(t, "sequential", sequentialPipeline.DisplayName)
sequentialPipelineVersion, err := s.pipelineClient.CreatePipelineVersion(
&params.CreatePipelineVersionParams{
PipelineID: sequentialPipeline.PipelineID,
Body: &model.V2beta1PipelineVersion{
DisplayName: "sequential-v1",
Description: "1st version of sequential pipeline",
PipelineID: sequentialPipeline.PipelineID,
PackageURL: &model.V2beta1URL{
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml",
},
},
})
require.Nil(t, err)
assert.Equal(t, "sequential-v1", sequentialPipelineVersion.DisplayName)
assert.Equal(t, "1st version of sequential pipeline", sequentialPipelineVersion.Description)
assert.Equal(t, sequentialPipeline.PipelineID, sequentialPipelineVersion.PipelineID)
assert.Equal(t, "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml", sequentialPipelineVersion.PackageURL.PipelineURL)
/* ---------- Upload pipelines zip ---------- */
time.Sleep(1 * time.Second)
argumentUploadPipeline, err := s.pipelineUploadClient.UploadFile(
"../resources/arguments.pipeline.zip", &upload_params.UploadPipelineParams{Name: util.StringPointer("zip-arguments-parameters")})
require.Nil(t, err)
assert.Equal(t, "zip-arguments-parameters", argumentUploadPipeline.DisplayName)
/* ---------- Import pipeline tarball by URL ---------- */
time.Sleep(1 * time.Second)
argumentUrlPipeline, err := s.pipelineClient.Create(&params.CreatePipelineParams{
Body: &model.V2beta1Pipeline{DisplayName: "arguments.pipeline.zip"},
})
require.Nil(t, err)
argumentUrlPipelineVersion, err := s.pipelineClient.CreatePipelineVersion(
&params.CreatePipelineVersionParams{
PipelineID: argumentUrlPipeline.PipelineID,
Body: &model.V2beta1PipelineVersion{
DisplayName: "argumentUrl-v1",
Description: "1st version of argument url pipeline",
PipelineID: sequentialPipeline.PipelineID,
PackageURL: &model.V2beta1URL{
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip",
},
},
})
require.Nil(t, err)
assert.Equal(t, "argumentUrl-v1", argumentUrlPipelineVersion.DisplayName)
assert.Equal(t, "1st version of argument url pipeline", argumentUrlPipelineVersion.Description)
assert.Equal(t, argumentUrlPipeline.PipelineID, argumentUrlPipelineVersion.PipelineID)
assert.Equal(t, "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip", argumentUrlPipelineVersion.PackageURL.PipelineURL)
/* ---------- Verify list pipeline works ---------- */
pipelines, totalSize, _, err := s.pipelineClient.List(&params.ListPipelinesParams{})
require.Nil(t, err)
assert.Equal(t, 5, len(pipelines))
assert.Equal(t, 5, totalSize)
for _, p := range pipelines {
// Sampling one of the pipelines and verify the result is expected.
if p.DisplayName == "arguments-parameters.yaml" {
assert.NotNil(t, *p)
assert.NotNil(t, p.CreatedAt)
}
}
/* ---------- Verify list pipeline sorted by names ---------- */
listFirstPagePipelines, totalSize, nextPageToken, err := s.pipelineClient.List(
&params.ListPipelinesParams{PageSize: util.Int32Pointer(2), SortBy: util.StringPointer("name")})
require.Nil(t, err)
assert.Equal(t, 2, len(listFirstPagePipelines))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "arguments-parameters.yaml", listFirstPagePipelines[0].DisplayName)
assert.Equal(t, "arguments.pipeline.zip", listFirstPagePipelines[1].DisplayName)
assert.NotEmpty(t, nextPageToken)
listSecondPagePipelines, totalSize, nextPageToken, err := s.pipelineClient.List(
&params.ListPipelinesParams{PageToken: util.StringPointer(nextPageToken), PageSize: util.Int32Pointer(3), SortBy: util.StringPointer("name")})
require.Nil(t, err)
assert.Equal(t, 3, len(listSecondPagePipelines))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "hello-world.yaml", listSecondPagePipelines[0].DisplayName)
assert.Equal(t, "sequential", listSecondPagePipelines[1].DisplayName)
assert.Equal(t, "zip-arguments-parameters", listSecondPagePipelines[2].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- Verify list pipeline sorted by creation time ---------- */
listFirstPagePipelines, totalSize, nextPageToken, err = s.pipelineClient.List(
&params.ListPipelinesParams{PageSize: util.Int32Pointer(3), SortBy: util.StringPointer("created_at")})
require.Nil(t, err)
assert.Equal(t, 3, len(listFirstPagePipelines))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "hello-world.yaml", listFirstPagePipelines[0].DisplayName)
assert.Equal(t, "arguments-parameters.yaml", listFirstPagePipelines[1].DisplayName)
assert.Equal(t, "sequential", listFirstPagePipelines[2].DisplayName)
assert.NotEmpty(t, nextPageToken)
listSecondPagePipelines, totalSize, nextPageToken, err = s.pipelineClient.List(
&params.ListPipelinesParams{PageToken: util.StringPointer(nextPageToken), PageSize: util.Int32Pointer(3), SortBy: util.StringPointer("created_at")})
require.Nil(t, err)
assert.Equal(t, 2, len(listSecondPagePipelines))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "zip-arguments-parameters", listSecondPagePipelines[0].DisplayName)
assert.Equal(t, "arguments.pipeline.zip", listSecondPagePipelines[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- List pipelines sort by unsupported description field. Should fail. ---------- */
_, _, _, err = s.pipelineClient.List(&params.ListPipelinesParams{
PageSize: util.Int32Pointer(2), SortBy: util.StringPointer("unknownfield"),
})
assert.NotNil(t, err)
/* ---------- List pipelines sorted by names descend order ---------- */
listFirstPagePipelines, totalSize, nextPageToken, err = s.pipelineClient.List(
&params.ListPipelinesParams{PageSize: util.Int32Pointer(3), SortBy: util.StringPointer("name desc")})
require.Nil(t, err)
assert.Equal(t, 3, len(listFirstPagePipelines))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "zip-arguments-parameters", listFirstPagePipelines[0].DisplayName)
assert.Equal(t, "sequential", listFirstPagePipelines[1].DisplayName)
assert.Equal(t, "hello-world.yaml", listFirstPagePipelines[2].DisplayName)
assert.NotEmpty(t, nextPageToken)
listSecondPagePipelines, totalSize, nextPageToken, err = s.pipelineClient.List(&params.ListPipelinesParams{
PageToken: util.StringPointer(nextPageToken), PageSize: util.Int32Pointer(3), SortBy: util.StringPointer("name desc"),
})
require.Nil(t, err)
assert.Equal(t, 2, len(listSecondPagePipelines))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "arguments.pipeline.zip", listSecondPagePipelines[0].DisplayName)
assert.Equal(t, "arguments-parameters.yaml", listSecondPagePipelines[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- Verify get pipeline works ---------- */
pipeline, err := s.pipelineClient.Get(&params.GetPipelineParams{PipelineID: argumentYAMLPipeline.PipelineID})
require.Nil(t, err)
assert.NotNil(t, *pipeline)
assert.NotNil(t, pipeline.CreatedAt)
assert.Equal(t, "arguments-parameters.yaml", pipeline.DisplayName)
}
func TestPipelineAPI(t *testing.T) {
suite.Run(t, new(PipelineApiTest))
}
func (s *PipelineApiTest) TearDownSuite() {
if *runIntegrationTests {
if !*isDevMode {
s.cleanUp()
}
}
}
func (s *PipelineApiTest) cleanUp() {
test.DeleteAllPipelines(s.pipelineClient, s.T())
}

347
backend/test/v2/integration/pipeline_version_api_test.go Normal file
View File

@ -0,0 +1,347 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"encoding/json"
"io/ioutil"
"strings"
"testing"
"time"
"github.com/golang/glog"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client/pipeline_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_model"
upload_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
"github.com/kubeflow/pipelines/backend/src/common/util"
test "github.com/kubeflow/pipelines/backend/test/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"sigs.k8s.io/yaml"
)
// This test suit tests various methods to import pipeline to pipeline system, including
// - upload yaml file
// - upload tarball file
// - providing YAML file url
// - Providing tarball file url
type PipelineVersionApiTest struct {
suite.Suite
namespace string
pipelineClient *api_server.PipelineClient
pipelineUploadClient *api_server.PipelineUploadClient
}
// Check the namespace have ML job installed and ready
func (s *PipelineVersionApiTest) SetupTest() {
if !*runIntegrationTests {
s.T().SkipNow()
return
}
if !*isDevMode {
err := test.WaitForReady(*namespace, *initializeTimeout)
if err != nil {
glog.Exitf("Failed to initialize test. Error: %s", err.Error())
}
}
var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
var newPipelineClient func() (*api_server.PipelineClient, error)
if *isKubeflowMode {
s.namespace = *namespace
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewKubeflowInClusterPipelineUploadClient(s.namespace, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewKubeflowInClusterPipelineClient(s.namespace, *isDebugMode)
}
} else {
clientConfig := test.GetClientConfig(*namespace)
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewPipelineUploadClient(clientConfig, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewPipelineClient(clientConfig, *isDebugMode)
}
}
var err error
s.pipelineUploadClient, err = newPipelineUploadClient()
if err != nil {
glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
}
s.pipelineClient, err = newPipelineClient()
if err != nil {
glog.Exitf("Failed to get pipeline client. Error: %s", err.Error())
}
s.cleanUp()
}
func (s *PipelineVersionApiTest) TestPipelineSpec() {
t := s.T()
test.DeleteAllPipelines(s.pipelineClient, t)
/* ---------- Upload a pipeline YAML ---------- */
pipelineParams := upload_params.NewUploadPipelineParams()
pipelineName := "test_pipeline"
pipelineParams.SetName(&pipelineName)
pipeline, err := s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", pipelineParams)
require.Nil(t, err)
assert.Equal(t, "test_pipeline", pipeline.DisplayName)
/* ---------- Get pipeline id ---------- */
pipelines, totalSize, _, err := s.pipelineClient.List(&params.ListPipelinesParams{})
require.Nil(t, err)
assert.Equal(t, 1, len(pipelines))
assert.Equal(t, 1, totalSize)
pipelineId := pipelines[0].PipelineID
/* ---------- Upload a pipeline version YAML under test_pipeline ---------- */
time.Sleep(1 * time.Second)
pipelineVersionParams := upload_params.NewUploadPipelineVersionParams()
pipelineVersionParams.SetPipelineid(&pipelineId)
argumentYAMLPipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion("../resources/arguments-parameters.yaml", pipelineVersionParams)
require.Nil(t, err)
assert.Equal(t, "arguments-parameters.yaml", argumentYAMLPipelineVersion.DisplayName)
/* ---------- Upload the same pipeline version again. Should fail due to name uniqueness ---------- */
time.Sleep(1 * time.Second)
_, err = s.pipelineUploadClient.UploadPipelineVersion("../resources/arguments-parameters.yaml", upload_params.NewUploadPipelineVersionParams())
require.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to upload pipeline version")
/* ---------- Import pipeline version YAML by URL ---------- */
time.Sleep(1 * time.Second)
sequentialPipelineVersion, err := s.pipelineClient.CreatePipelineVersion(&params.CreatePipelineVersionParams{
PipelineID: pipelineId,
Body: &pipeline_model.V2beta1PipelineVersion{
DisplayName: "sequential",
PackageURL: &pipeline_model.V2beta1URL{
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml",
},
PipelineID: pipelineId,
},
})
require.Nil(t, err)
assert.Equal(t, "sequential", sequentialPipelineVersion.DisplayName)
/* ---------- Upload pipeline version zip ---------- */
time.Sleep(1 * time.Second)
argumentUploadPipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion(
"../resources/arguments.pipeline.zip", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("zip-arguments-parameters"),
Pipelineid: util.StringPointer(pipelineId),
})
require.Nil(t, err)
assert.Equal(t, "zip-arguments-parameters", argumentUploadPipelineVersion.DisplayName)
/* ---------- Import pipeline tarball by URL ---------- */
time.Sleep(1 * time.Second)
argumentUrlPipelineVersion, err := s.pipelineClient.CreatePipelineVersion(&params.CreatePipelineVersionParams{
PipelineID: pipelineId,
Body: &pipeline_model.V2beta1PipelineVersion{
DisplayName: "arguments",
PackageURL: &pipeline_model.V2beta1URL{
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip",
},
PipelineID: pipelineId,
},
})
require.Nil(t, err)
assert.Equal(t, "arguments", argumentUrlPipelineVersion.DisplayName)
/* ---------- Verify list pipeline version works ---------- */
pipelineVersions, totalSize, _, err := s.pipelineClient.ListPipelineVersions(&params.ListPipelineVersionsParams{
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 5, len(pipelineVersions))
assert.Equal(t, 5, totalSize)
for _, p := range pipelineVersions {
assert.NotNil(t, *p)
assert.NotNil(t, p.CreatedAt)
assert.Contains(t, []string{"test_pipeline" /*default version created with pipeline*/, "sequential", "arguments", "arguments-parameters.yaml", "zip-arguments-parameters"}, p.DisplayName)
}
/* ---------- Verify list pipeline sorted by names ---------- */
listFirstPagePipelineVersions, totalSize, nextPageToken, err := s.pipelineClient.ListPipelineVersions(
&params.ListPipelineVersionsParams{
PageSize: util.Int32Pointer(3),
SortBy: util.StringPointer("name"),
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 3, len(listFirstPagePipelineVersions))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "arguments", listFirstPagePipelineVersions[0].DisplayName)
assert.Equal(t, "arguments-parameters.yaml", listFirstPagePipelineVersions[1].DisplayName)
assert.Equal(t, "sequential", listFirstPagePipelineVersions[2].DisplayName)
assert.NotEmpty(t, nextPageToken)
listSecondPagePipelineVersions, totalSize, nextPageToken, err := s.pipelineClient.ListPipelineVersions(
&params.ListPipelineVersionsParams{
PageToken: util.StringPointer(nextPageToken),
PageSize: util.Int32Pointer(3),
SortBy: util.StringPointer("name"),
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 2, len(listSecondPagePipelineVersions))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "test_pipeline", listSecondPagePipelineVersions[0].DisplayName)
assert.Equal(t, "zip-arguments-parameters", listSecondPagePipelineVersions[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- Verify list pipeline version sorted by creation time ---------- */
listFirstPagePipelineVersions, totalSize, nextPageToken, err = s.pipelineClient.ListPipelineVersions(
&params.ListPipelineVersionsParams{
PageSize: util.Int32Pointer(3),
SortBy: util.StringPointer("created_at"),
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 3, len(listFirstPagePipelineVersions))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "test_pipeline", listFirstPagePipelineVersions[0].DisplayName)
assert.Equal(t, "arguments-parameters.yaml", listFirstPagePipelineVersions[1].DisplayName)
assert.Equal(t, "sequential", listFirstPagePipelineVersions[2].DisplayName)
assert.NotEmpty(t, nextPageToken)
listSecondPagePipelineVersions, totalSize, nextPageToken, err = s.pipelineClient.ListPipelineVersions(
&params.ListPipelineVersionsParams{
PageToken: util.StringPointer(nextPageToken),
PageSize: util.Int32Pointer(3),
SortBy: util.StringPointer("created_at"),
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 2, len(listSecondPagePipelineVersions))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "zip-arguments-parameters", listSecondPagePipelineVersions[0].DisplayName)
assert.Equal(t, "arguments", listSecondPagePipelineVersions[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- List pipeline versions sort by unsupported description field. Should fail. ---------- */
_, _, _, err = s.pipelineClient.ListPipelineVersions(&params.ListPipelineVersionsParams{
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("unknownfield"),
PipelineID: pipelineId,
})
assert.NotNil(t, err)
/* ---------- List pipeline versions sorted by names descend order ---------- */
listFirstPagePipelineVersions, totalSize, nextPageToken, err = s.pipelineClient.ListPipelineVersions(
&params.ListPipelineVersionsParams{
PageSize: util.Int32Pointer(3),
SortBy: util.StringPointer("name desc"),
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 3, len(listFirstPagePipelineVersions))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "zip-arguments-parameters", listFirstPagePipelineVersions[0].DisplayName)
assert.Equal(t, "test_pipeline", listFirstPagePipelineVersions[1].DisplayName)
assert.Equal(t, "sequential", listFirstPagePipelineVersions[2].DisplayName)
assert.NotEmpty(t, nextPageToken)
listSecondPagePipelineVersions, totalSize, nextPageToken, err = s.pipelineClient.ListPipelineVersions(
&params.ListPipelineVersionsParams{
PageToken: util.StringPointer(nextPageToken),
PageSize: util.Int32Pointer(3),
SortBy: util.StringPointer("name desc"),
PipelineID: pipelineId,
})
require.Nil(t, err)
assert.Equal(t, 2, len(listSecondPagePipelineVersions))
assert.Equal(t, 5, totalSize)
assert.Equal(t, "arguments-parameters.yaml", listSecondPagePipelineVersions[0].DisplayName)
assert.Equal(t, "arguments", listSecondPagePipelineVersions[1].DisplayName)
assert.Empty(t, nextPageToken)
/* ---------- Verify get pipeline version works ---------- */
pipelineVersion, err := s.pipelineClient.GetPipelineVersion(&params.GetPipelineVersionParams{PipelineID: argumentUrlPipelineVersion.PipelineID, PipelineVersionID: argumentUrlPipelineVersion.PipelineVersionID})
require.Nil(t, err)
assert.Equal(t, pipelineVersion.DisplayName, "arguments")
assert.NotNil(t, pipelineVersion.CreatedAt)
/* ---------- Verify pipeline spec ---------- */
bytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
require.Nil(t, err)
expected_bytes, err := yaml.YAMLToJSON(bytes)
require.Nil(t, err)
actual_bytes, err := json.Marshal(pipelineVersion.PipelineSpec)
require.Nil(t, err)
// Override pipeline name, then compare
assert.Equal(t, string(expected_bytes), strings.Replace(string(actual_bytes), "pipeline/test_pipeline", "whalesay", 1))
}
func (s *PipelineVersionApiTest) TestV2Spec() {
t := s.T()
test.DeleteAllPipelines(s.pipelineClient, t)
/* ---------- Upload a pipeline YAML ---------- */
pipelineParams := upload_params.NewUploadPipelineParams()
pipelineName := "test_v2_pipeline"
pipelineParams.SetName(&pipelineName)
pipeline, err := s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", pipelineParams)
require.Nil(t, err)
assert.Equal(t, "test_v2_pipeline", pipeline.DisplayName)
/* ---------- Upload a pipeline version with v2 pipeline spec YAML ---------- */
time.Sleep(1 * time.Second)
v2Version, err := s.pipelineUploadClient.UploadPipelineVersion(
"../resources/hello-world.yaml", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("hello-world"),
Pipelineid: util.StringPointer(pipeline.PipelineID),
})
require.Nil(t, err)
assert.Equal(t, "hello-world", v2Version.DisplayName)
/* ---------- Verify pipeline spec ---------- */
bytes, err := ioutil.ReadFile("../resources/hello-world.yaml")
require.Nil(t, err)
expected_bytes, err := yaml.YAMLToJSON(bytes)
require.Nil(t, err)
actual_bytes, err := json.Marshal(v2Version.PipelineSpec)
require.Nil(t, err)
// Override pipeline name, then compare
assert.Equal(t, string(expected_bytes), strings.Replace(string(actual_bytes), "pipeline/test_v2_pipeline", "whalesay", 1))
}
func TestPipelineVersionAPI(t *testing.T) {
suite.Run(t, new(PipelineVersionApiTest))
}
func (s *PipelineVersionApiTest) TearDownSuite() {
if *runIntegrationTests {
if !*isDevMode {
s.cleanUp()
}
}
}
func (s *PipelineVersionApiTest) cleanUp() {
// Delete pipelines and their pipeline versions
test.DeleteAllPipelines(s.pipelineClient, s.T())
}

720
backend/test/v2/integration/recurring_run_api_test.go Normal file
View File

@ -0,0 +1,720 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"context"
"fmt"
"io/ioutil"
"strings"
"testing"
"time"
"github.com/eapache/go-resiliency/retrier"
"github.com/go-openapi/strfmt"
"github.com/golang/glog"
experiment_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client/pipeline_service"
upload_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
recurring_run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_client/recurring_run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_model"
run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_model"
"github.com/kubeflow/pipelines/backend/src/apiserver/client"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
"github.com/kubeflow/pipelines/backend/src/common/util"
test "github.com/kubeflow/pipelines/backend/test/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"google.golang.org/protobuf/types/known/structpb"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
)
const (
second = 1
minute = 60 * second
hour = 60 * minute
)
type RecurringRunApiTestSuite struct {
suite.Suite
namespace string
resourceNamespace string
experimentClient *api_server.ExperimentClient
pipelineClient *api_server.PipelineClient
pipelineUploadClient *api_server.PipelineUploadClient
runClient *api_server.RunClient
recurringRunClient *api_server.RecurringRunClient
swfClient client.SwfClientInterface
}
// Check the namespace have ML pipeline installed and ready
func (s *RecurringRunApiTestSuite) SetupTest() {
if !*runIntegrationTests {
s.T().SkipNow()
return
}
if !*isDevMode {
err := test.WaitForReady(*namespace, *initializeTimeout)
if err != nil {
glog.Exitf("Failed to initialize test. Error: %s", err.Error())
}
}
s.namespace = *namespace
var newExperimentClient func() (*api_server.ExperimentClient, error)
var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
var newPipelineClient func() (*api_server.PipelineClient, error)
var newRunClient func() (*api_server.RunClient, error)
var newRecurringRunClient func() (*api_server.RecurringRunClient, error)
if *isKubeflowMode {
s.resourceNamespace = *resourceNamespace
newExperimentClient = func() (*api_server.ExperimentClient, error) {
return api_server.NewKubeflowInClusterExperimentClient(s.namespace, *isDebugMode)
}
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewKubeflowInClusterPipelineUploadClient(s.namespace, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewKubeflowInClusterPipelineClient(s.namespace, *isDebugMode)
}
newRunClient = func() (*api_server.RunClient, error) {
return api_server.NewKubeflowInClusterRunClient(s.namespace, *isDebugMode)
}
newRecurringRunClient = func() (*api_server.RecurringRunClient, error) {
return api_server.NewKubeflowInClusterRecurringRunClient(s.namespace, *isDebugMode)
}
} else {
clientConfig := test.GetClientConfig(*namespace)
newExperimentClient = func() (*api_server.ExperimentClient, error) {
return api_server.NewExperimentClient(clientConfig, *isDebugMode)
}
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewPipelineUploadClient(clientConfig, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewPipelineClient(clientConfig, *isDebugMode)
}
newRunClient = func() (*api_server.RunClient, error) {
return api_server.NewRunClient(clientConfig, *isDebugMode)
}
newRecurringRunClient = func() (*api_server.RecurringRunClient, error) {
return api_server.NewRecurringRunClient(clientConfig, *isDebugMode)
}
}
var err error
s.experimentClient, err = newExperimentClient()
if err != nil {
glog.Exitf("Failed to get experiment client. Error: %v", err)
}
s.pipelineUploadClient, err = newPipelineUploadClient()
if err != nil {
glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
}
s.pipelineClient, err = newPipelineClient()
if err != nil {
glog.Exitf("Failed to get pipeline client. Error: %s", err.Error())
}
s.runClient, err = newRunClient()
if err != nil {
glog.Exitf("Failed to get run client. Error: %s", err.Error())
}
s.recurringRunClient, err = newRecurringRunClient()
if err != nil {
glog.Exitf("Failed to get recurringRun client. Error: %s", err.Error())
}
s.swfClient = client.NewScheduledWorkflowClientOrFatal(time.Second*30, util.ClientParameters{QPS: 5, Burst: 10})
s.cleanUp()
}
func (s *RecurringRunApiTestSuite) TestRecurringRunApis() {
t := s.T()
/* ---------- Upload pipelines YAML ---------- */
helloWorldPipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", upload_params.NewUploadPipelineParams())
assert.Nil(t, err)
/* ---------- Upload pipeline version YAML ---------- */
time.Sleep(1 * time.Second)
helloWorldPipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion(
"../resources/hello-world.yaml", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("hello-world-version"),
Pipelineid: util.StringPointer(helloWorldPipeline.PipelineID),
})
assert.Nil(t, err)
/* ---------- Create a new hello world experiment ---------- */
experiment := test.MakeExperiment("hello world experiment", "", s.resourceNamespace)
helloWorldExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
/* ---------- Create a new hello world recurringRun by specifying pipeline ID ---------- */
createRecurringRunRequest := &recurring_run_params.CreateRecurringRunParams{Body: &recurring_run_model.V2beta1RecurringRun{
DisplayName: "hello world",
Description: "this is hello world",
ExperimentID: helloWorldExperiment.ExperimentID,
PipelineVersionReference: &recurring_run_model.V2beta1PipelineVersionReference{
PipelineID: helloWorldPipelineVersion.PipelineID,
PipelineVersionID: helloWorldPipelineVersion.PipelineVersionID,
},
MaxConcurrency: 10,
Mode: recurring_run_model.RecurringRunModeENABLE,
}}
helloWorldRecurringRun, err := s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
s.checkHelloWorldRecurringRun(t, helloWorldRecurringRun, helloWorldExperiment.ExperimentID, helloWorldPipelineVersion.PipelineID, helloWorldPipelineVersion.PipelineVersionID)
/* ---------- Get hello world recurringRun ---------- */
helloWorldRecurringRun, err = s.recurringRunClient.Get(&recurring_run_params.GetRecurringRunParams{RecurringRunID: helloWorldRecurringRun.RecurringRunID})
assert.Nil(t, err)
s.checkHelloWorldRecurringRun(t, helloWorldRecurringRun, helloWorldExperiment.ExperimentID, helloWorldPipelineVersion.PipelineID, helloWorldPipelineVersion.PipelineVersionID)
/* ---------- Create a new argument parameter experiment ---------- */
experiment = test.MakeExperiment("argument parameter experiment", "", s.resourceNamespace)
argParamsExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
/* ---------- Create a new argument parameter recurringRun by uploading workflow manifest ---------- */
// Make sure the recurringRun is created at least 1 second later than the first one,
// because sort by created_at has precision of 1 second.
time.Sleep(1 * time.Second)
argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
assert.Nil(t, err)
pipeline_spec := &structpb.Struct{}
err = yaml.Unmarshal(argParamsBytes, pipeline_spec)
assert.Nil(t, err)
createRecurringRunRequest = &recurring_run_params.CreateRecurringRunParams{Body: &recurring_run_model.V2beta1RecurringRun{
DisplayName: "argument parameter",
Description: "this is argument parameter",
ExperimentID: argParamsExperiment.ExperimentID,
PipelineSpec: pipeline_spec,
RuntimeConfig: &recurring_run_model.V2beta1RuntimeConfig{
Parameters: map[string]interface{}{
"param1": "goodbye",
"param2": "world",
},
},
MaxConcurrency: 10,
Mode: recurring_run_model.RecurringRunModeENABLE,
}}
argParamsRecurringRun, err := s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
s.checkArgParamsRecurringRun(t, argParamsRecurringRun, argParamsExperiment.ExperimentID)
/* ---------- List all the recurringRuns. Both recurringRuns should be returned ---------- */
recurringRuns, totalSize, _, err := test.ListAllRecurringRuns(s.recurringRunClient, s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 2, totalSize, "Incorrect total number of recurring runs in the namespace %v", s.resourceNamespace)
assert.Equal(t, 2, len(recurringRuns), "Incorrect total number of recurring runs in the namespace %v", s.resourceNamespace)
/* ---------- List the recurringRuns, paginated, sort by creation time ---------- */
recurringRuns, totalSize, nextPageToken, err := test.ListRecurringRuns(
s.recurringRunClient,
&recurring_run_params.ListRecurringRunsParams{
PageSize: util.Int32Pointer(1),
SortBy: util.StringPointer("created_at"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, len(recurringRuns))
assert.Equal(t, 2, totalSize)
assert.Equal(t, "hello world", recurringRuns[0].DisplayName)
recurringRuns, totalSize, _, err = test.ListRecurringRuns(
s.recurringRunClient,
&recurring_run_params.ListRecurringRunsParams{
PageSize: util.Int32Pointer(1),
PageToken: util.StringPointer(nextPageToken),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, len(recurringRuns))
assert.Equal(t, 2, totalSize)
assert.Equal(t, "argument parameter", recurringRuns[0].DisplayName)
/* ---------- List the recurringRuns, paginated, sort by name ---------- */
recurringRuns, totalSize, nextPageToken, err = test.ListRecurringRuns(
s.recurringRunClient,
&recurring_run_params.ListRecurringRunsParams{
PageSize: util.Int32Pointer(1),
SortBy: util.StringPointer("name"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 2, totalSize)
assert.Equal(t, 1, len(recurringRuns))
assert.Equal(t, "argument parameter", recurringRuns[0].DisplayName)
recurringRuns, totalSize, _, err = test.ListRecurringRuns(
s.recurringRunClient,
&recurring_run_params.ListRecurringRunsParams{
PageSize: util.Int32Pointer(1),
SortBy: util.StringPointer("name"),
PageToken: util.StringPointer(nextPageToken),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 2, totalSize)
assert.Equal(t, 1, len(recurringRuns))
assert.Equal(t, "hello world", recurringRuns[0].DisplayName)
/* ---------- List the recurringRuns, sort by unsupported field ---------- */
recurringRuns, _, _, err = test.ListRecurringRuns(
s.recurringRunClient,
&recurring_run_params.ListRecurringRunsParams{
PageSize: util.Int32Pointer(2),
SortBy: util.StringPointer("unknown"),
},
s.resourceNamespace)
assert.NotNil(t, err)
assert.Equal(t, len(recurringRuns), 0)
/* ---------- List recurringRuns for hello world experiment. One recurringRun should be returned ---------- */
recurringRuns, totalSize, _, err = s.recurringRunClient.List(&recurring_run_params.ListRecurringRunsParams{
ExperimentID: util.StringPointer(helloWorldExperiment.ExperimentID),
})
assert.Nil(t, err)
assert.Equal(t, 1, len(recurringRuns))
assert.Equal(t, 1, totalSize)
assert.Equal(t, "hello world", recurringRuns[0].DisplayName)
/* ---------- List the recurringRuns, filtered by created_at, only return the previous two recurringRuns ---------- */
time.Sleep(5 * time.Second) // Sleep for 5 seconds to make sure the previous recurringRuns are created at a different timestamp
filterTime := time.Now().Unix()
time.Sleep(5 * time.Second)
createRecurringRunRequestNew := &recurring_run_params.CreateRecurringRunParams{Body: &recurring_run_model.V2beta1RecurringRun{
DisplayName: "new hello world recurringRun",
Description: "this is a new hello world",
ExperimentID: helloWorldExperiment.ExperimentID,
PipelineVersionReference: &recurring_run_model.V2beta1PipelineVersionReference{
PipelineID: helloWorldPipelineVersion.PipelineID,
PipelineVersionID: helloWorldPipelineVersion.PipelineVersionID,
},
MaxConcurrency: 10,
Mode: recurring_run_model.RecurringRunModeDISABLE,
}}
_, err = s.recurringRunClient.Create(createRecurringRunRequestNew)
assert.Nil(t, err)
// Check total number of recurringRuns to be 3
recurringRuns, totalSize, _, err = test.ListAllRecurringRuns(s.recurringRunClient, s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 3, totalSize)
assert.Equal(t, 3, len(recurringRuns))
// Check number of filtered recurringRuns finished before filterTime to be 2
recurringRuns, totalSize, _, err = test.ListRecurringRuns(
s.recurringRunClient,
&recurring_run_params.ListRecurringRunsParams{
Filter: util.StringPointer(`{"predicates": [{"key": "created_at", "operation": "LESS_THAN", "string_value": "` + fmt.Sprint(filterTime) + `"}]}`),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 2, len(recurringRuns))
assert.Equal(t, 2, totalSize)
// The scheduledWorkflow CRD would create the run and it synced to the DB by persistent agent.
// This could take a few seconds to finish.
/* ---------- Check run for hello world recurringRun ---------- */
if err := retrier.New(retrier.ConstantBackoff(8, 5*time.Second), nil).Run(func() error {
runs, totalSize, _, err := s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(helloWorldExperiment.ExperimentID),
})
if err != nil {
return err
}
if len(runs) != 1 {
return fmt.Errorf("expected runs to be length 1, got: %v", len(runs))
}
if totalSize != 1 {
return fmt.Errorf("expected total size 1, got: %v", totalSize)
}
helloWorldRun := runs[0]
return s.checkHelloWorldRun(helloWorldRun, helloWorldExperiment.ExperimentID, helloWorldRecurringRun.RecurringRunID)
}); err != nil {
assert.Nil(t, err)
}
/* ---------- Check run for argument parameter recurringRun ---------- */
if err := retrier.New(retrier.ConstantBackoff(8, 5*time.Second), nil).Run(func() error {
runs, totalSize, _, err := s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(argParamsExperiment.ExperimentID),
})
if err != nil {
return err
}
if len(runs) != 1 {
return fmt.Errorf("expected runs to be length 1, got: %v", len(runs))
}
if totalSize != 1 {
return fmt.Errorf("expected total size 1, got: %v", totalSize)
}
argParamsRun := runs[0]
return s.checkArgParamsRun(argParamsRun, argParamsExperiment.ExperimentID, argParamsRecurringRun.RecurringRunID)
}); err != nil {
assert.Nil(t, err)
}
}
func (s *RecurringRunApiTestSuite) TestRecurringRunApis_noCatchupOption() {
t := s.T()
/* ---------- Upload pipelines YAML ---------- */
pipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", upload_params.NewUploadPipelineParams())
assert.Nil(t, err)
/* ---------- Upload pipeline version YAML ---------- */
time.Sleep(1 * time.Second)
helloWorldPipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion(
"../resources/hello-world.yaml", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("hello-world-version"),
Pipelineid: util.StringPointer(pipeline.PipelineID),
})
assert.Nil(t, err)
/* ---------- Create a periodic recurringRun with start and end date in the past and catchup = true ---------- */
experiment := test.MakeExperiment("periodic catchup true", "", s.resourceNamespace)
periodicCatchupTrueExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
recurringRun := recurringRunInThePastForTwoMinutes(recurringRunOptions{
pipelineId: helloWorldPipelineVersion.PipelineID,
pipelineVersionId: helloWorldPipelineVersion.PipelineVersionID,
experimentId: periodicCatchupTrueExperiment.ExperimentID,
periodic: true,
})
recurringRun.DisplayName = "periodic-catchup-true-"
recurringRun.Description = "A recurringRun with NoCatchup=false will backfill each past interval when behind schedule."
recurringRun.NoCatchup = false // This is the key difference.
createRecurringRunRequest := &recurring_run_params.CreateRecurringRunParams{Body: recurringRun}
_, err = s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
/* -------- Create another periodic recurringRun with start and end date in the past but catchup = false ------ */
experiment = test.MakeExperiment("periodic catchup false", "", s.resourceNamespace)
periodicCatchupFalseExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
recurringRun = recurringRunInThePastForTwoMinutes(recurringRunOptions{
pipelineId: helloWorldPipelineVersion.PipelineID,
pipelineVersionId: helloWorldPipelineVersion.PipelineVersionID,
experimentId: periodicCatchupFalseExperiment.ExperimentID,
periodic: true,
})
recurringRun.DisplayName = "periodic-catchup-false-"
recurringRun.Description = "A recurringRun with NoCatchup=true only schedules the last interval when behind schedule."
recurringRun.NoCatchup = true // This is the key difference.
createRecurringRunRequest = &recurring_run_params.CreateRecurringRunParams{Body: recurringRun}
_, err = s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
/* ---------- Create a cron recurringRun with start and end date in the past and catchup = true ---------- */
experiment = test.MakeExperiment("cron catchup true", "", s.resourceNamespace)
cronCatchupTrueExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
recurringRun = recurringRunInThePastForTwoMinutes(recurringRunOptions{
pipelineId: helloWorldPipelineVersion.PipelineID,
pipelineVersionId: helloWorldPipelineVersion.PipelineVersionID,
experimentId: cronCatchupTrueExperiment.ExperimentID,
periodic: false,
})
recurringRun.DisplayName = "cron-catchup-true-"
recurringRun.Description = "A recurringRun with NoCatchup=false will backfill each past interval when behind schedule."
recurringRun.NoCatchup = false // This is the key difference.
createRecurringRunRequest = &recurring_run_params.CreateRecurringRunParams{Body: recurringRun}
_, err = s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
/* -------- Create another cron recurringRun with start and end date in the past but catchup = false ------ */
experiment = test.MakeExperiment("cron catchup false", "", s.resourceNamespace)
cronCatchupFalseExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
recurringRun = recurringRunInThePastForTwoMinutes(recurringRunOptions{
pipelineId: helloWorldPipelineVersion.PipelineID,
pipelineVersionId: helloWorldPipelineVersion.PipelineVersionID,
experimentId: cronCatchupFalseExperiment.ExperimentID,
periodic: false,
})
recurringRun.DisplayName = "cron-catchup-false-"
recurringRun.Description = "A recurringRun with NoCatchup=true only schedules the last interval when behind schedule."
recurringRun.NoCatchup = true // This is the key difference.
createRecurringRunRequest = &recurring_run_params.CreateRecurringRunParams{Body: recurringRun}
_, err = s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
// The scheduledWorkflow CRD would create the run and it is synced to the DB by persistent agent.
// This could take a few seconds to finish.
/* ---------- Assert number of runs when catchup = true ---------- */
if err := retrier.New(retrier.ConstantBackoff(8, 5*time.Second), nil).Run(func() error {
_, runsWhenCatchupTrue, _, err := s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(periodicCatchupTrueExperiment.ExperimentID),
})
if err != nil {
return err
}
if runsWhenCatchupTrue != 2 {
return fmt.Errorf("expected runsWhenCatchupTrue with periodic schedule to be 2, got: %v", runsWhenCatchupTrue)
}
_, runsWhenCatchupTrue, _, err = s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(cronCatchupTrueExperiment.ExperimentID),
})
if err != nil {
return err
}
if runsWhenCatchupTrue != 2 {
return fmt.Errorf("expected runsWhenCatchupTrue with cron schedule to be 2, got: %v", runsWhenCatchupTrue)
}
return nil
}); err != nil {
assert.Nil(t, err)
}
/* ---------- Assert number of runs when catchup = false ---------- */
if err := retrier.New(retrier.ConstantBackoff(8, 5*time.Second), nil).Run(func() error {
_, runsWhenCatchupFalse, _, err := s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(periodicCatchupFalseExperiment.ExperimentID),
})
if err != nil {
return err
}
if runsWhenCatchupFalse != 1 {
return fmt.Errorf("expected runsWhenCatchupFalse with periodic schedule to be 1, got: %v", runsWhenCatchupFalse)
}
_, runsWhenCatchupFalse, _, err = s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(cronCatchupFalseExperiment.ExperimentID),
})
if err != nil {
return err
}
if runsWhenCatchupFalse != 1 {
return fmt.Errorf("expected runsWhenCatchupFalse with cron schedule to be 1, got: %v", runsWhenCatchupFalse)
}
return nil
}); err != nil {
assert.Nil(t, err)
}
}
func (s *RecurringRunApiTestSuite) checkHelloWorldRecurringRun(t *testing.T, recurringRun *recurring_run_model.V2beta1RecurringRun, experimentID string, pipelineId string, pipelineVersionId string) {
expectedRecurringRun := &recurring_run_model.V2beta1RecurringRun{
RecurringRunID: recurringRun.RecurringRunID,
DisplayName: "hello world",
Description: "this is hello world",
ServiceAccount: test.GetDefaultPipelineRunnerServiceAccount(*isKubeflowMode),
PipelineSpec: recurringRun.PipelineSpec,
ExperimentID: experimentID,
PipelineVersionReference: &recurring_run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineId,
PipelineVersionID: pipelineVersionId,
},
MaxConcurrency: 10,
Mode: recurring_run_model.RecurringRunModeENABLE,
Namespace: recurringRun.Namespace,
CreatedAt: recurringRun.CreatedAt,
UpdatedAt: recurringRun.UpdatedAt,
Trigger: recurringRun.Trigger,
Status: recurringRun.Status,
}
assert.Equal(t, expectedRecurringRun, recurringRun)
}
func (s *RecurringRunApiTestSuite) checkArgParamsRecurringRun(t *testing.T, recurringRun *recurring_run_model.V2beta1RecurringRun, experimentID string) {
expectedRecurringRun := &recurring_run_model.V2beta1RecurringRun{
RecurringRunID: recurringRun.RecurringRunID,
DisplayName: "argument parameter",
Description: "this is argument parameter",
ServiceAccount: test.GetDefaultPipelineRunnerServiceAccount(*isKubeflowMode),
PipelineSpec: recurringRun.PipelineSpec,
RuntimeConfig: &recurring_run_model.V2beta1RuntimeConfig{
Parameters: map[string]interface{}{
"param1": "goodbye",
"param2": "world",
},
},
ExperimentID: experimentID,
MaxConcurrency: 10,
Mode: recurring_run_model.RecurringRunModeENABLE,
Namespace: recurringRun.Namespace,
CreatedAt: recurringRun.CreatedAt,
UpdatedAt: recurringRun.UpdatedAt,
Trigger: recurringRun.Trigger,
Status: recurringRun.Status,
}
assert.Equal(t, expectedRecurringRun, recurringRun)
}
func (s *RecurringRunApiTestSuite) TestRecurringRunApis_SwfNotFound() {
t := s.T()
/* ---------- Upload pipelines YAML ---------- */
pipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", upload_params.NewUploadPipelineParams())
assert.Nil(t, err)
pipelineVersions, totalSize, _, err := s.pipelineClient.ListPipelineVersions(&params.ListPipelineVersionsParams{
PipelineID: pipeline.PipelineID,
})
assert.Nil(t, err)
assert.Equal(t, totalSize, 1)
/* ---------- Create a new hello world recurringRun by specifying pipeline ID ---------- */
experiment := test.MakeExperiment("test-swf-not-found experiment", "", s.resourceNamespace)
swfNotFoundExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
createRecurringRunRequest := &recurring_run_params.CreateRecurringRunParams{Body: &recurring_run_model.V2beta1RecurringRun{
DisplayName: "test-swf-not-found",
ExperimentID: swfNotFoundExperiment.ExperimentID,
PipelineVersionReference: &recurring_run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineVersions[0].PipelineID,
PipelineVersionID: pipelineVersions[0].PipelineVersionID,
},
MaxConcurrency: 10,
Mode: recurring_run_model.RecurringRunModeDISABLE,
}}
recurringRun, err := s.recurringRunClient.Create(createRecurringRunRequest)
assert.Nil(t, err)
// Delete all ScheduledWorkflow custom resources to simulate the situation
// that after reinstalling KFP with managed storage, only KFP DB is kept,
// but all KFP custom resources are gone.
swfNamespace := s.namespace
if s.resourceNamespace != "" {
swfNamespace = s.resourceNamespace
}
err = s.swfClient.ScheduledWorkflow(swfNamespace).DeleteCollection(context.Background(), &v1.DeleteOptions{}, v1.ListOptions{})
assert.Nil(t, err)
err = s.recurringRunClient.Delete(&recurring_run_params.DeleteRecurringRunParams{RecurringRunID: recurringRun.RecurringRunID})
assert.Nil(t, err)
/* ---------- Get recurringRun ---------- */
_, err = s.recurringRunClient.Get(&recurring_run_params.GetRecurringRunParams{RecurringRunID: recurringRun.RecurringRunID})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "status 404")
}
func (s *RecurringRunApiTestSuite) checkHelloWorldRun(run *run_model.V2beta1Run, experimentID string, recurringRunID string) error {
if !strings.Contains(run.DisplayName, "helloworld") {
return fmt.Errorf("expected: %+v got: %+v", "helloworld", run.DisplayName)
}
if run.ExperimentID != experimentID {
return fmt.Errorf("expected: %+v got: %+v", experimentID, run.ExperimentID)
}
if run.RecurringRunID != recurringRunID {
return fmt.Errorf("expected: %+v got: %+v", recurringRunID, run.RecurringRunID)
}
return nil
}
func (s *RecurringRunApiTestSuite) checkArgParamsRun(run *run_model.V2beta1Run, experimentID, recurringRunID string) error {
if !strings.Contains(run.DisplayName, "argumentparameter") {
return fmt.Errorf("expected: %+v got: %+v", "argumentparameter", run.DisplayName)
}
if run.ExperimentID != experimentID {
return fmt.Errorf("expected: %+v got: %+v", experimentID, run.ExperimentID)
}
if run.RecurringRunID != recurringRunID {
return fmt.Errorf("expected: %+v got: %+v", recurringRunID, run.RecurringRunID)
}
return nil
}
func TestRecurringRunApi(t *testing.T) {
suite.Run(t, new(RecurringRunApiTestSuite))
}
func (s *RecurringRunApiTestSuite) TearDownSuite() {
if *runIntegrationTests {
if !*isDevMode {
s.cleanUp()
}
}
}
/** ======== the following are util functions ========= **/
func (s *RecurringRunApiTestSuite) cleanUp() {
test.DeleteAllRuns(s.runClient, s.resourceNamespace, s.T())
test.DeleteAllRecurringRuns(s.recurringRunClient, s.resourceNamespace, s.T())
test.DeleteAllPipelines(s.pipelineClient, s.T())
test.DeleteAllExperiments(s.experimentClient, s.resourceNamespace, s.T())
}
func defaultV2beta1RecurringRun(pipelineId, pipelineVersionId, experimentId string) *recurring_run_model.V2beta1RecurringRun {
return &recurring_run_model.V2beta1RecurringRun{
DisplayName: "default-pipeline-name",
Description: "This is a default pipeline",
ExperimentID: experimentId,
PipelineVersionReference: &recurring_run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineId,
PipelineVersionID: pipelineVersionId,
},
MaxConcurrency: 10,
NoCatchup: false,
Trigger: &recurring_run_model.V2beta1Trigger{
PeriodicSchedule: &recurring_run_model.V2beta1PeriodicSchedule{
StartTime: strfmt.NewDateTime(),
EndTime: strfmt.NewDateTime(),
IntervalSecond: 60,
},
},
Mode: recurring_run_model.RecurringRunModeENABLE,
}
}
type recurringRunOptions struct {
pipelineId, pipelineVersionId, experimentId string
periodic bool
}
func recurringRunInThePastForTwoMinutes(options recurringRunOptions) *recurring_run_model.V2beta1RecurringRun {
startTime := strfmt.DateTime(time.Unix(10*hour, 0))
endTime := strfmt.DateTime(time.Unix(10*hour+2*minute, 0))
recurringRun := defaultV2beta1RecurringRun(options.pipelineId, options.pipelineVersionId, options.experimentId)
if options.periodic {
recurringRun.Trigger = &recurring_run_model.V2beta1Trigger{
PeriodicSchedule: &recurring_run_model.V2beta1PeriodicSchedule{
StartTime: startTime,
EndTime: endTime,
IntervalSecond: 60, // Runs every 1 minute.
},
}
} else {
recurringRun.Trigger = &recurring_run_model.V2beta1Trigger{
CronSchedule: &recurring_run_model.V2beta1CronSchedule{
StartTime: startTime,
EndTime: endTime,
Cron: "0 * * * * ?", // Runs every 1 minute.
},
}
}
return recurringRun
}

444
backend/test/v2/integration/run_api_test.go Normal file
View File

@ -0,0 +1,444 @@
// Copyright 2018-2023 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration
import (
"encoding/json"
"fmt"
"io/ioutil"
"testing"
"time"
"github.com/golang/glog"
experiment_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
upload_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_model"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
"github.com/kubeflow/pipelines/backend/src/common/util"
test "github.com/kubeflow/pipelines/backend/test/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"google.golang.org/protobuf/types/known/structpb"
"sigs.k8s.io/yaml"
)
type RunApiTestSuite struct {
suite.Suite
namespace string
resourceNamespace string
experimentClient *api_server.ExperimentClient
pipelineClient *api_server.PipelineClient
pipelineUploadClient *api_server.PipelineUploadClient
runClient *api_server.RunClient
}
// Check the namespace have ML pipeline installed and ready
func (s *RunApiTestSuite) SetupTest() {
if !*runIntegrationTests {
s.T().SkipNow()
return
}
if !*isDevMode {
err := test.WaitForReady(*namespace, *initializeTimeout)
if err != nil {
glog.Exitf("Failed to initialize test. Error: %s", err.Error())
}
}
s.namespace = *namespace
var newExperimentClient func() (*api_server.ExperimentClient, error)
var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
var newPipelineClient func() (*api_server.PipelineClient, error)
var newRunClient func() (*api_server.RunClient, error)
if *isKubeflowMode {
s.resourceNamespace = *resourceNamespace
newExperimentClient = func() (*api_server.ExperimentClient, error) {
return api_server.NewKubeflowInClusterExperimentClient(s.namespace, *isDebugMode)
}
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewKubeflowInClusterPipelineUploadClient(s.namespace, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewKubeflowInClusterPipelineClient(s.namespace, *isDebugMode)
}
newRunClient = func() (*api_server.RunClient, error) {
return api_server.NewKubeflowInClusterRunClient(s.namespace, *isDebugMode)
}
} else {
clientConfig := test.GetClientConfig(*namespace)
newExperimentClient = func() (*api_server.ExperimentClient, error) {
return api_server.NewExperimentClient(clientConfig, *isDebugMode)
}
newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
return api_server.NewPipelineUploadClient(clientConfig, *isDebugMode)
}
newPipelineClient = func() (*api_server.PipelineClient, error) {
return api_server.NewPipelineClient(clientConfig, *isDebugMode)
}
newRunClient = func() (*api_server.RunClient, error) {
return api_server.NewRunClient(clientConfig, *isDebugMode)
}
}
var err error
s.experimentClient, err = newExperimentClient()
if err != nil {
glog.Exitf("Failed to get experiment client. Error: %v", err)
}
s.pipelineUploadClient, err = newPipelineUploadClient()
if err != nil {
glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error())
}
s.pipelineClient, err = newPipelineClient()
if err != nil {
glog.Exitf("Failed to get pipeline client. Error: %s", err.Error())
}
s.runClient, err = newRunClient()
if err != nil {
glog.Exitf("Failed to get run client. Error: %s", err.Error())
}
s.cleanUp()
}
func (s *RunApiTestSuite) TestRunApis() {
t := s.T()
/* ---------- Upload pipelines YAML ---------- */
helloWorldPipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", upload_params.NewUploadPipelineParams())
assert.Nil(t, err)
/* ---------- Upload a pipeline version YAML under helloWorldPipeline ---------- */
time.Sleep(1 * time.Second)
helloWorldPipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion(
"../resources/hello-world.yaml", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("hello-world-version"),
Pipelineid: util.StringPointer(helloWorldPipeline.PipelineID),
})
assert.Nil(t, err)
/* ---------- Create a new hello world experiment ---------- */
experiment := test.MakeExperiment("hello world experiment", "", s.resourceNamespace)
helloWorldExperiment, err := s.experimentClient.Create(&experiment_params.CreateExperimentParams{Body: experiment})
assert.Nil(t, err)
/* ---------- Create a new hello world run by specifying pipeline version ID ---------- */
createRunRequest := &run_params.CreateRunParams{Body: &run_model.V2beta1Run{
DisplayName: "hello world",
Description: "this is hello world",
ExperimentID: helloWorldExperiment.ExperimentID,
PipelineVersionReference: &run_model.V2beta1PipelineVersionReference{
PipelineID: helloWorldPipelineVersion.PipelineID,
PipelineVersionID: helloWorldPipelineVersion.PipelineVersionID,
},
}}
helloWorldRunDetail, err := s.runClient.Create(createRunRequest)
assert.Nil(t, err)
s.checkHelloWorldRunDetail(t, helloWorldRunDetail, helloWorldExperiment.ExperimentID, helloWorldPipelineVersion.PipelineID, helloWorldPipelineVersion.PipelineVersionID)
/* ---------- Get hello world run ---------- */
helloWorldRunDetail, err = s.runClient.Get(&run_params.GetRunParams{RunID: helloWorldRunDetail.RunID})
assert.Nil(t, err)
s.checkHelloWorldRunDetail(t, helloWorldRunDetail, helloWorldExperiment.ExperimentID, helloWorldPipelineVersion.PipelineID, helloWorldPipelineVersion.PipelineVersionID)
/* ---------- Create a new argument parameter experiment ---------- */
createExperimentRequest := &experiment_params.CreateExperimentParams{
Body: test.MakeExperiment("argument parameter experiment", "", s.resourceNamespace),
}
argParamsExperiment, err := s.experimentClient.Create(createExperimentRequest)
assert.Nil(t, err)
/* ---------- Create a new argument parameter run by uploading workflow manifest ---------- */
argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
assert.Nil(t, err)
pipeline_spec := &structpb.Struct{}
err = yaml.Unmarshal(argParamsBytes, pipeline_spec)
assert.Nil(t, err)
createRunRequest = &run_params.CreateRunParams{Body: &run_model.V2beta1Run{
DisplayName: "argument parameter",
Description: "this is argument parameter",
PipelineSpec: pipeline_spec,
RuntimeConfig: &run_model.V2beta1RuntimeConfig{
Parameters: map[string]interface{}{
"param1": "goodbye",
"param2": "world",
},
},
ExperimentID: argParamsExperiment.ExperimentID,
}}
argParamsRunDetail, err := s.runClient.Create(createRunRequest)
assert.Nil(t, err)
s.checkArgParamsRunDetail(t, argParamsRunDetail, argParamsExperiment.ExperimentID)
/* ---------- List all the runs. Both runs should be returned ---------- */
runs, totalSize, _, err := test.ListAllRuns(s.runClient, s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 2, len(runs))
assert.Equal(t, 2, totalSize)
/* ---------- List the runs, paginated, sorted by creation time ---------- */
runs, totalSize, nextPageToken, err := test.ListRuns(
s.runClient,
&run_params.ListRunsParams{
PageSize: util.Int32Pointer(1),
SortBy: util.StringPointer("created_at"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, len(runs))
assert.Equal(t, 2, totalSize)
/* TODO(issues/1762): fix the following flaky assertion. */
/* assert.Equal(t, "hello world", runs[0].Name) */
runs, totalSize, _, err = test.ListRuns(
s.runClient,
&run_params.ListRunsParams{
PageSize: util.Int32Pointer(1),
PageToken: util.StringPointer(nextPageToken),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, len(runs))
assert.Equal(t, 2, totalSize)
/* TODO(issues/1762): fix the following flaky assertion. */
/* assert.Equal(t, "argument parameter", runs[0].Name) */
/* ---------- List the runs, paginated, sort by name ---------- */
runs, totalSize, nextPageToken, err = test.ListRuns(
s.runClient,
&run_params.ListRunsParams{
PageSize: util.Int32Pointer(1),
SortBy: util.StringPointer("name"),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, len(runs))
assert.Equal(t, 2, totalSize)
assert.Equal(t, "argument parameter", runs[0].DisplayName)
runs, totalSize, _, err = test.ListRuns(
s.runClient,
&run_params.ListRunsParams{
PageSize: util.Int32Pointer(1),
SortBy: util.StringPointer("name"),
PageToken: util.StringPointer(nextPageToken),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 1, len(runs))
assert.Equal(t, 2, totalSize)
assert.Equal(t, "hello world", runs[0].DisplayName)
/* ---------- List the runs, sort by unsupported field ---------- */
_, _, _, err = test.ListRuns(
s.runClient,
&run_params.ListRunsParams{PageSize: util.Int32Pointer(2), SortBy: util.StringPointer("unknownfield")},
s.resourceNamespace)
assert.NotNil(t, err)
/* ---------- List runs for hello world experiment. One run should be returned ---------- */
runs, totalSize, _, err = s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(helloWorldExperiment.ExperimentID),
})
assert.Nil(t, err)
assert.Equal(t, 1, len(runs))
assert.Equal(t, 1, totalSize)
assert.Equal(t, "hello world", runs[0].DisplayName)
/* ---------- List the runs, filtered by created_at, only return the previous two runs ---------- */
time.Sleep(5 * time.Second) // Sleep for 5 seconds to make sure the previous runs are created at a different timestamp
filterTime := time.Now().Unix()
time.Sleep(5 * time.Second)
// Create a new run
createRunRequest.Body.DisplayName = "argument parameter 2"
_, err = s.runClient.Create(createRunRequest)
assert.Nil(t, err)
// Check total number of runs is 3
runs, totalSize, _, err = test.ListAllRuns(s.runClient, s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 3, len(runs))
assert.Equal(t, 3, totalSize)
// Check number of filtered runs created before filterTime to be 2
runs, totalSize, _, err = test.ListRuns(
s.runClient,
&run_params.ListRunsParams{
Filter: util.StringPointer(`{"predicates": [{"key": "created_at", "operation": "LESS_THAN", "string_value": "` + fmt.Sprint(filterTime) + `"}]}`),
},
s.resourceNamespace)
assert.Nil(t, err)
assert.Equal(t, 2, len(runs))
assert.Equal(t, 2, totalSize)
/* ---------- Archive a run ------------*/
err = s.runClient.Archive(&run_params.ArchiveRunParams{
RunID: helloWorldRunDetail.RunID,
})
assert.Nil(t, err)
/* ---------- List runs for hello world experiment. The same run should still be returned, but should be archived ---------- */
runs, totalSize, _, err = s.runClient.List(&run_params.ListRunsParams{
ExperimentID: util.StringPointer(helloWorldExperiment.ExperimentID),
})
assert.Nil(t, err)
assert.Equal(t, 1, len(runs))
assert.Equal(t, 1, totalSize)
assert.Equal(t, "hello world", runs[0].DisplayName)
assert.Equal(t, run_model.V2beta1RunStorageStateARCHIVED, runs[0].StorageState)
/* ---------- Upload long-running pipeline YAML ---------- */
longRunningPipeline, err := s.pipelineUploadClient.UploadFile("../resources/long-running.yaml", upload_params.NewUploadPipelineParamsWithTimeout(350))
assert.Nil(t, err)
/* ---------- Upload a long-running pipeline version YAML under longRunningPipeline ---------- */
time.Sleep(1 * time.Second)
longRunningPipelineVersion, err := s.pipelineUploadClient.UploadPipelineVersion("../resources/long-running.yaml", &upload_params.UploadPipelineVersionParams{
Name: util.StringPointer("long-running-version"),
Pipelineid: util.StringPointer(longRunningPipeline.PipelineID),
})
assert.Nil(t, err)
/* ---------- Create a new long-running run by specifying pipeline ID ---------- */
createLongRunningRunRequest := &run_params.CreateRunParams{Body: &run_model.V2beta1Run{
DisplayName: "long running",
Description: "this pipeline will run long enough for us to manually terminate it before it finishes",
ExperimentID: helloWorldExperiment.ExperimentID,
PipelineVersionReference: &run_model.V2beta1PipelineVersionReference{
PipelineID: longRunningPipelineVersion.PipelineID,
PipelineVersionID: longRunningPipelineVersion.PipelineVersionID,
},
}}
longRunningRun, err := s.runClient.Create(createLongRunningRunRequest)
assert.Nil(t, err)
/* ---------- Terminate the long-running run ------------*/
err = s.runClient.Terminate(&run_params.TerminateRunParams{
RunID: longRunningRun.RunID,
})
assert.Nil(t, err)
/* ---------- Get long-running run ---------- */
longRunningRun, err = s.runClient.Get(&run_params.GetRunParams{RunID: longRunningRun.RunID})
assert.Nil(t, err)
s.checkTerminatedRunDetail(t, longRunningRun, helloWorldExperiment.ExperimentID, longRunningPipelineVersion.PipelineID, longRunningPipelineVersion.PipelineVersionID)
}
func (s *RunApiTestSuite) checkTerminatedRunDetail(t *testing.T, run *run_model.V2beta1Run, experimentId string, pipelineId string, pipelineVersionId string) {
expectedRun := &run_model.V2beta1Run{
RunID: run.RunID,
DisplayName: "long running",
Description: "this pipeline will run long enough for us to manually terminate it before it finishes",
State: run_model.V2beta1RuntimeStateCANCELING,
StateHistory: run.StateHistory,
StorageState: run.StorageState,
ServiceAccount: test.GetDefaultPipelineRunnerServiceAccount(*isKubeflowMode),
PipelineSpec: run.PipelineSpec,
ExperimentID: experimentId,
PipelineVersionReference: &run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineId,
PipelineVersionID: pipelineVersionId,
},
CreatedAt: run.CreatedAt,
ScheduledAt: run.ScheduledAt,
FinishedAt: run.FinishedAt,
}
assert.Equal(t, expectedRun, run)
}
func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, run *run_model.V2beta1Run, experimentId string, pipelineId string, pipelineVersionId string) {
expectedRun := &run_model.V2beta1Run{
RunID: run.RunID,
DisplayName: "hello world",
Description: "this is hello world",
State: run.State,
StateHistory: run.StateHistory,
StorageState: run.StorageState,
ServiceAccount: test.GetDefaultPipelineRunnerServiceAccount(*isKubeflowMode),
PipelineSpec: run.PipelineSpec,
ExperimentID: experimentId,
PipelineVersionReference: &run_model.V2beta1PipelineVersionReference{
PipelineID: pipelineId,
PipelineVersionID: pipelineVersionId,
},
CreatedAt: run.CreatedAt,
ScheduledAt: run.ScheduledAt,
FinishedAt: run.FinishedAt,
}
assert.Equal(t, expectedRun, run)
}
func (s *RunApiTestSuite) checkArgParamsRunDetail(t *testing.T, run *run_model.V2beta1Run, experimentId string) {
// Compare the pipeline spec first.
argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
assert.Nil(t, err)
// pipeline_spec := &structpb.Struct{}
// err = yaml.Unmarshal(argParamsBytes, pipeline_spec)
// assert.Nil(t, err)
expected_bytes, err := yaml.YAMLToJSON(argParamsBytes)
assert.Nil(t, err)
actual_bytes, err := json.Marshal(run.PipelineSpec)
assert.Nil(t, err)
assert.Equal(t, string(expected_bytes), string(actual_bytes))
expectedRun := &run_model.V2beta1Run{
RunID: run.RunID,
DisplayName: "argument parameter",
Description: "this is argument parameter",
State: run.State,
StateHistory: run.StateHistory,
StorageState: run.StorageState,
ServiceAccount: test.GetDefaultPipelineRunnerServiceAccount(*isKubeflowMode),
PipelineSpec: run.PipelineSpec,
RuntimeConfig: &run_model.V2beta1RuntimeConfig{
Parameters: map[string]interface{}{
"param1": "goodbye",
"param2": "world",
},
},
ExperimentID: experimentId,
CreatedAt: run.CreatedAt,
ScheduledAt: run.ScheduledAt,
FinishedAt: run.FinishedAt,
}
assert.Equal(t, expectedRun, run)
}
func TestRunApi(t *testing.T) {
suite.Run(t, new(RunApiTestSuite))
}
func (s *RunApiTestSuite) TearDownSuite() {
if *runIntegrationTests {
if !*isDevMode {
s.cleanUp()
}
}
}
func (s *RunApiTestSuite) cleanUp() {
/* ---------- Clean up ---------- */
test.DeleteAllRuns(s.runClient, s.resourceNamespace, s.T())
test.DeleteAllPipelines(s.pipelineClient, s.T())
test.DeleteAllExperiments(s.experimentClient, s.resourceNamespace, s.T())
}

31
backend/test/v2/resources/arguments-parameters.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp import dsl, compiler
@dsl.container_component
def whalesay(param1: str, param2: str):
return dsl.ContainerSpec(
image='docker/whalesay:latest',
command=['cowsay'],
args=[f'{param1}-{param2}'],
)
if __name__ == '__main__':
compiler.Compiler().compile(
whalesay,
package_path=__file__.replace('.py', '.yaml'),
pipeline_parameters={'param1': 'hello'})

50
backend/test/v2/resources/arguments-parameters.yaml Normal file
View File

@ -0,0 +1,50 @@
# PIPELINE DEFINITION
# Name: whalesay
# Inputs:
# param1: str [Default: 'hello']
# param2: str
components:
comp-whalesay:
executorLabel: exec-whalesay
inputDefinitions:
parameters:
param1:
parameterType: STRING
param2:
parameterType: STRING
deploymentSpec:
executors:
exec-whalesay:
container:
args:
- '{{$.inputs.parameters[''param1'']}}-{{$.inputs.parameters[''param2'']}}'
command:
- cowsay
image: docker/whalesay:latest
pipelineInfo:
name: whalesay
root:
dag:
tasks:
whalesay:
cachingOptions:
enableCache: true
componentRef:
name: comp-whalesay
inputs:
parameters:
param1:
componentInputParameter: param1
param2:
componentInputParameter: param2
taskInfo:
name: whalesay
inputDefinitions:
parameters:
param1:
defaultValue: hello
parameterType: STRING
param2:
parameterType: STRING
schemaVersion: 2.1.0
sdkVersion: kfp-2.0.0-beta.13

BIN
backend/test/v2/resources/arguments.pipeline.zip Normal file
View File

Binary file not shown.

29
backend/test/v2/resources/hello-world.py Normal file
View File

@ -0,0 +1,29 @@
# Copyright 2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp import dsl, compiler
@dsl.container_component
def whalesay():
return dsl.ContainerSpec(
image='docker/whalesay:latest',
command=['cowsay'],
args=['hello world'],
)
if __name__ == '__main__':
compiler.Compiler().compile(
whalesay, package_path=__file__.replace('.py', '.yaml'))

28
backend/test/v2/resources/hello-world.yaml Normal file
View File

@ -0,0 +1,28 @@
# PIPELINE DEFINITION
# Name: whalesay
components:
comp-whalesay:
executorLabel: exec-whalesay
deploymentSpec:
executors:
exec-whalesay:
container:
args:
- hello world
command:
- cowsay
image: docker/whalesay:latest
pipelineInfo:
name: whalesay
root:
dag:
tasks:
whalesay:
cachingOptions:
enableCache: true
componentRef:
name: comp-whalesay
taskInfo:
name: whalesay
schemaVersion: 2.1.0
sdkVersion: kfp-2.0.0-beta.13

35
backend/test/v2/resources/long-running.py Normal file
View File

@ -0,0 +1,35 @@
# Copyright 2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp import dsl, compiler
@dsl.container_component
def wait_op():
return dsl.ContainerSpec(
image='alpine:latest',
command=['sh', '-c'],
args=['echo step-1 sleeping for 5m; sleep 300; echo done1'],
)
@dsl.pipeline
def wait_awhile():
task1 = wait_op()
task2 = wait_op().after(task1)
if __name__ == '__main__':
compiler.Compiler().compile(
wait_awhile, package_path=__file__.replace('.py', '.yaml'))

48
backend/test/v2/resources/long-running.yaml Normal file
View File

@ -0,0 +1,48 @@
# PIPELINE DEFINITION
# Name: wait-awhile
components:
comp-wait-op:
executorLabel: exec-wait-op
comp-wait-op-2:
executorLabel: exec-wait-op-2
deploymentSpec:
executors:
exec-wait-op:
container:
args:
- echo step-1 sleeping for 5m; sleep 300; echo done1
command:
- sh
- -c
image: alpine:latest
exec-wait-op-2:
container:
args:
- echo step-1 sleeping for 5m; sleep 300; echo done1
command:
- sh
- -c
image: alpine:latest
pipelineInfo:
name: wait-awhile
root:
dag:
tasks:
wait-op:
cachingOptions:
enableCache: true
componentRef:
name: comp-wait-op
taskInfo:
name: wait-op
wait-op-2:
cachingOptions:
enableCache: true
componentRef:
name: comp-wait-op-2
dependentTasks:
- wait-op
taskInfo:
name: wait-op-2
schemaVersion: 2.1.0
sdkVersion: kfp-2.0.0-beta.13

43
backend/test/v2/resources/sequential.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright 2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp import dsl, compiler
@dsl.container_component
def download(url: str, downloaded: dsl.OutputPath(str)):
return dsl.ContainerSpec(
image='google/cloud-sdk',
command=['sh', '-c'],
args=[f'gsutil cp {url} {downloaded}'],
)
@dsl.container_component
def echo(downloaded: str):
return dsl.ContainerSpec(
image='library/bash',
command=['sh', '-c'],
args=[f'echo {downloaded}'],
)
@dsl.pipeline
def sequential(url: str):
echo(downloaded=download(url=url).outputs['downloaded'])
if __name__ == '__main__':
compiler.Compiler().compile(
sequential, package_path=__file__.replace('.py', '.yaml'))

76
backend/test/v2/resources/sequential.yaml Normal file
View File

@ -0,0 +1,76 @@
# PIPELINE DEFINITION
# Name: sequential
# Inputs:
# url: str
components:
comp-download:
executorLabel: exec-download
inputDefinitions:
parameters:
url:
parameterType: STRING
outputDefinitions:
parameters:
downloaded:
parameterType: STRING
comp-echo:
executorLabel: exec-echo
inputDefinitions:
parameters:
downloaded:
parameterType: STRING
deploymentSpec:
executors:
exec-download:
container:
args:
- gsutil cp {{$.inputs.parameters['url']}} {{$.outputs.parameters['downloaded'].output_file}}
command:
- sh
- -c
image: google/cloud-sdk
exec-echo:
container:
args:
- echo {{$.inputs.parameters['downloaded']}}
command:
- sh
- -c
image: library/bash
pipelineInfo:
name: sequential
root:
dag:
tasks:
download:
cachingOptions:
enableCache: true
componentRef:
name: comp-download
inputs:
parameters:
url:
componentInputParameter: url
taskInfo:
name: download
echo:
cachingOptions:
enableCache: true
componentRef:
name: comp-echo
dependentTasks:
- download
inputs:
parameters:
downloaded:
taskOutputParameter:
outputParameterKey: downloaded
producerTask: download
taskInfo:
name: echo
inputDefinitions:
parameters:
url:
parameterType: STRING
schemaVersion: 2.1.0
sdkVersion: kfp-2.0.0-beta.13

117
backend/test/v2/test_utils.go
View File

@ -22,8 +22,14 @@ import (
"time"
"github.com/cenkalti/backoff"
experimentparams "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
experiment_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_client/experiment_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/experiment_model"
pipeline_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client/pipeline_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_model"
recurring_run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_client/recurring_run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/recurring_run_model"
run_params "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service"
"github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_model"
api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
@ -61,11 +67,19 @@ func GetClientConfig(namespace string) clientcmd.ClientConfig {
&overrides, os.Stdin)
}
func ListAllExperiment(client *api_server.ExperimentClient, namespace string) ([]*experiment_model.V2beta1Experiment, int, string, error) {
return ListExperiment(client, &experimentparams.ListExperimentsParams{}, namespace)
func GetDefaultPipelineRunnerServiceAccount(isKubeflowMode bool) string {
if isKubeflowMode {
return "default-editor"
} else {
return "pipeline-runner"
}
}
func ListExperiment(client *api_server.ExperimentClient, parameters *experimentparams.ListExperimentsParams, namespace string) ([]*experiment_model.V2beta1Experiment, int, string, error) {
func ListAllExperiment(client *api_server.ExperimentClient, namespace string) ([]*experiment_model.V2beta1Experiment, int, string, error) {
return ListExperiment(client, &experiment_params.ListExperimentsParams{}, namespace)
}
func ListExperiment(client *api_server.ExperimentClient, parameters *experiment_params.ListExperimentsParams, namespace string) ([]*experiment_model.V2beta1Experiment, int, string, error) {
if namespace != "" {
parameters.Namespace = &namespace
}
@ -77,7 +91,100 @@ func DeleteAllExperiments(client *api_server.ExperimentClient, namespace string,
assert.Nil(t, err)
for _, e := range experiments {
if e.DisplayName != "Default" {
assert.Nil(t, client.Delete(&experimentparams.DeleteExperimentParams{ExperimentID: e.ExperimentID}))
assert.Nil(t, client.Delete(&experiment_params.DeleteExperimentParams{ExperimentID: e.ExperimentID}))
}
}
}
func MakeExperiment(name string, description string, namespace string) *experiment_model.V2beta1Experiment {
experiment := &experiment_model.V2beta1Experiment{
DisplayName: name,
Description: description,
}
if namespace != "" {
experiment.Namespace = namespace
}
return experiment
}
func ListRuns(client *api_server.RunClient, parameters *run_params.ListRunsParams, namespace string) ([]*run_model.V2beta1Run, int, string, error) {
if namespace != "" {
parameters.Namespace = &namespace
}
return client.List(parameters)
}
func ListAllRuns(client *api_server.RunClient, namespace string) ([]*run_model.V2beta1Run, int, string, error) {
parameters := &run_params.ListRunsParams{}
return ListRuns(client, parameters, namespace)
}
func DeleteAllRuns(client *api_server.RunClient, namespace string, t *testing.T) {
runs, _, _, err := ListAllRuns(client, namespace)
assert.Nil(t, err)
for _, r := range runs {
assert.Nil(t, client.Delete(&run_params.DeleteRunParams{RunID: r.RunID}))
}
}
func ListRecurringRuns(client *api_server.RecurringRunClient, parameters *recurring_run_params.ListRecurringRunsParams, namespace string) ([]*recurring_run_model.V2beta1RecurringRun, int, string, error) {
if namespace != "" {
parameters.Namespace = &namespace
}
return client.List(parameters)
}
func ListAllRecurringRuns(client *api_server.RecurringRunClient, namespace string) ([]*recurring_run_model.V2beta1RecurringRun, int, string, error) {
return ListRecurringRuns(client, &recurring_run_params.ListRecurringRunsParams{}, namespace)
}
func DeleteAllRecurringRuns(client *api_server.RecurringRunClient, namespace string, t *testing.T) {
recurringRuns, _, _, err := ListAllRecurringRuns(client, namespace)
assert.Nil(t, err)
for _, r := range recurringRuns {
assert.Nil(t, client.Delete(&recurring_run_params.DeleteRecurringRunParams{RecurringRunID: r.RecurringRunID}))
}
}
func ListPipelineVersions(client *api_server.PipelineClient, pipelineId string) (
[]*pipeline_model.V2beta1PipelineVersion, int, string, error,
) {
parameters := &pipeline_params.ListPipelineVersionsParams{PipelineID: pipelineId}
return client.ListPipelineVersions(parameters)
}
func ListPipelines(client *api_server.PipelineClient) (
[]*pipeline_model.V2beta1Pipeline, int, string, error,
) {
parameters := &pipeline_params.ListPipelinesParams{}
return client.List(parameters)
}
func DeleteAllPipelineVersions(client *api_server.PipelineClient, t *testing.T, pipelineId string) {
pipelineVersions, _, _, err := ListPipelineVersions(client, pipelineId)
assert.Nil(t, err)
for _, pv := range pipelineVersions {
assert.Nil(t, client.DeletePipelineVersion(&pipeline_params.DeletePipelineVersionParams{PipelineID: pipelineId, PipelineVersionID: pv.PipelineVersionID}))
}
}
func DeleteAllPipelines(client *api_server.PipelineClient, t *testing.T) {
pipelines, _, _, err := ListPipelines(client)
assert.Nil(t, err)
deletedPipelines := make(map[string]bool)
for _, p := range pipelines {
deletedPipelines[p.PipelineID] = false
}
for pId, isRemoved := range deletedPipelines {
if !isRemoved {
DeleteAllPipelineVersions(client, t, pId)
deletedPipelines[pId] = true
}
assert.Nil(t, client.Delete(&pipeline_params.DeletePipelineParams{PipelineID: pId}))
}
for _, isRemoved := range deletedPipelines {
assert.True(t, isRemoved)
}
}

12
test/api-integration-test/run_test.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright 2018 The Kubeflow Authors
# Copyright 2018-2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -27,6 +27,7 @@ usage()
[--namespace k8s namespace where ml-pipelines is deployed. The tests run against the instance in this namespace]
[--run_upgrade_tests_preparation run preparation step of upgrade tests instead]
[--run_upgrade_tests_verification run verification step of upgrade tests instead]
[--test-v2-api run test using v2 API]
[-h help]"
}
@ -38,6 +39,9 @@ while [ "$1" != "" ]; do
--namespace ) shift
NAMESPACE=$1
;;
--test_v2_api )
TEST_V2_API=true
;;
--run_upgrade_tests_preparation )
UPGRADE_TESTS_PREPARATION=true
;;
@ -65,7 +69,11 @@ fi
GITHUB_REPO=kubeflow/pipelines
BASE_DIR=/go/src/github.com/${GITHUB_REPO}
JUNIT_TEST_RESULT=junit_ApiIntegrationTestOutput.xml
TEST_DIR=backend/test/integration
if [ -n "$TEST_V2_API" ]; then
TEST_DIR=backend/test/v2/integration
else
TEST_DIR=backend/test/integration
fi
cd "${BASE_DIR}/${TEST_DIR}"

22
test/e2e_test_gke_v2.yaml
View File

@ -111,6 +111,14 @@ spec:
value: "{{inputs.parameters.test-results-gcs-dir}}"
- name: api-integration-test-image
value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.api-integration-test-image-suffix}}"
- - name: run-api-integration-tests-v2
template: run-api-integration-tests-v2
arguments:
parameters:
- name: test-results-gcs-dir
value: "{{inputs.parameters.test-results-gcs-dir}}/v2"
- name: api-integration-test-image
value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.api-integration-test-image-suffix}}"
- - name: run-frontend-integration-tests
template: run-frontend-integration-tests
arguments:
@ -267,6 +275,20 @@ spec:
"--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}",
]
- name: run-api-integration-tests-v2
inputs:
parameters:
- name: test-results-gcs-dir
- name: api-integration-test-image
container:
image: "{{inputs.parameters.api-integration-test-image}}"
command:
- /go/src/github.com/kubeflow/pipelines/test/api-integration-test/run_test.sh
args: [
"--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}",
"--test_v2_api",
]
- name: run-frontend-integration-tests
inputs:
parameters: