kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add reference name to resource reference API proto (#1781) 

* add resource reference

* fix tests

* fix tests

* fix e2e test

* fix e2e

* fix test

* update api requirement

* fix tests

* Update job_api_test.go

* Update run_api_test.go

* Update setup.py

* Update deploy-kubeflow.sh

* fix tests

* Update deploy-kubeflow.sh
This commit is contained in:
IronPan 2019-08-13 03:15:11 -07:00 committed by GitHub
parent d710c7a0aa
commit 4e43750c9d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 347 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
backend/api/go_client/resource_reference.pb.go generated
View File

@ -55,7 +55,7 @@ func (x ResourceType) String() string {
return proto.EnumName(ResourceType_name, int32(x))
}
func (ResourceType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_resource_reference_7070bd2b275e38fb, []int{0}
return fileDescriptor_resource_reference_05b185b0f444334d, []int{0}
}
type Relationship int32
@ -81,7 +81,7 @@ func (x Relationship) String() string {
return proto.EnumName(Relationship_name, int32(x))
}
func (Relationship) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_resource_reference_7070bd2b275e38fb, []int{1}
return fileDescriptor_resource_reference_05b185b0f444334d, []int{1}
}
type ResourceKey struct {
@ -96,7 +96,7 @@ func (m *ResourceKey) Reset() { *m = ResourceKey{} }
func (m *ResourceKey) String() string { return proto.CompactTextString(m) }
func (*ResourceKey) ProtoMessage() {}
func (*ResourceKey) Descriptor() ([]byte, []int) {
return fileDescriptor_resource_reference_7070bd2b275e38fb, []int{0}
return fileDescriptor_resource_reference_05b185b0f444334d, []int{0}
}
func (m *ResourceKey) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_ResourceKey.Unmarshal(m, b)
@ -132,6 +132,7 @@ func (m *ResourceKey) GetId() string {
type ResourceReference struct {
Key *ResourceKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
Relationship Relationship `protobuf:"varint,2,opt,name=relationship,proto3,enum=api.Relationship" json:"relationship,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
@ -142,7 +143,7 @@ func (m *ResourceReference) Reset() { *m = ResourceReference{} }
func (m *ResourceReference) String() string { return proto.CompactTextString(m) }
func (*ResourceReference) ProtoMessage() {}
func (*ResourceReference) Descriptor() ([]byte, []int) {
return fileDescriptor_resource_reference_7070bd2b275e38fb, []int{1}
return fileDescriptor_resource_reference_05b185b0f444334d, []int{1}
}
func (m *ResourceReference) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_ResourceReference.Unmarshal(m, b)
@ -169,6 +170,13 @@ func (m *ResourceReference) GetKey() *ResourceKey {
return nil
}
func (m *ResourceReference) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *ResourceReference) GetRelationship() Relationship {
if m != nil {
return m.Relationship
@ -184,29 +192,30 @@ func init() {
}
func init() {
proto.RegisterFile("backend/api/resource_reference.proto", fileDescriptor_resource_reference_7070bd2b275e38fb)
proto.RegisterFile("backend/api/resource_reference.proto", fileDescriptor_resource_reference_05b185b0f444334d)
}
var fileDescriptor_resource_reference_7070bd2b275e38fb = []byte{
// 312 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6b, 0xf2, 0x30,
0x14, 0x47, 0x6d, 0xfd, 0xbe, 0x89, 0x57, 0x91, 0x18, 0x36, 0x70, 0x6f, 0x22, 0x1b, 0x88, 0x0f,
0x2d, 0x28, 0xbe, 0x4f, 0x5d, 0x60, 0xce, 0xad, 0x95, 0x58, 0x71, 0xdb, 0x4b, 0x69, 0xeb, 0x55,
0x83, 0x5d, 0x13, 0x62, 0x65, 0xf4, 0xbf, 0x1f, 0x96, 0x49, 0xf5, 0xf9, 0x1c, 0xce, 0x2f, 0xe4,
0xc2, 0x43, 0x18, 0x44, 0x7b, 0x4c, 0xd6, 0x76, 0xa0, 0x84, 0xad, 0xf1, 0x20, 0x8f, 0x3a, 0x42,
0x5f, 0xe3, 0x06, 0x35, 0x26, 0x11, 0x5a, 0x4a, 0xcb, 0x54, 0xd2, 0x72, 0xa0, 0x44, 0xe7, 0x19,
0x6a, 0xfc, 0x4f, 0x98, 0x61, 0x46, 0x1f, 0xe1, 0x5f, 0x9a, 0x29, 0x6c, 0x19, 0x6d, 0xa3, 0xdb,
0xe8, 0x37, 0xad, 0x40, 0x09, 0xeb, 0xcc, 0xbd, 0x4c, 0x21, 0xcf, 0x31, 0x6d, 0x80, 0x29, 0xd6,
0x2d, 0xb3, 0x6d, 0x74, 0xab, 0xdc, 0x14, 0xeb, 0x4e, 0x02, 0xcd, 0xb3, 0xc5, 0xcf, 0x2b, 0xb4,
0x03, 0xe5, 0x3d, 0x66, 0x79, 0xaa, 0xd6, 0x27, 0x57, 0xa9, 0x19, 0x66, 0xfc, 0x04, 0xe9, 0x10,
0xea, 0x1a, 0xe3, 0x20, 0x15, 0x32, 0x39, 0xec, 0x84, 0xca, 0x93, 0xc5, 0x6e, 0x01, 0xf8, 0x95,
0xd6, 0x1b, 0x43, 0xfd, 0xf2, 0x55, 0xf4, 0x1e, 0xee, 0x96, 0xce, 0xcc, 0x71, 0x57, 0x8e, 0xcf,
0xd9, 0xc2, 0x5d, 0xf2, 0x09, 0xf3, 0xbd, 0xcf, 0x39, 0x23, 0x25, 0xda, 0x00, 0x60, 0x1f, 0x73,
0xc6, 0xa7, 0xef, 0xcc, 0xf1, 0x88, 0x41, 0x2b, 0x50, 0x7e, 0x75, 0xc7, 0xc4, 0xec, 0x3d, 0x9d,
0x1a, 0x45, 0x93, 0xb6, 0xe0, 0xb6, 0x68, 0xbc, 0x8d, 0xbc, 0xa9, 0xeb, 0x2c, 0x5e, 0xa6, 0x73,
0x52, 0xa2, 0x55, 0xf8, 0xef, 0xae, 0x1c, 0xc6, 0x89, 0x41, 0x6b, 0x50, 0x99, 0x70, 0x36, 0xf2,
0x5c, 0x4e, 0xcc, 0xf1, 0xf0, 0x6b, 0xb0, 0x15, 0xe9, 0xee, 0x18, 0x5a, 0x91, 0xfc, 0xb6, 0xf7,
0xc7, 0x10, 0x37, 0xb1, 0xfc, 0xb1, 0x95, 0x50, 0x18, 0x8b, 0x04, 0x0f, 0xf6, 0xe5, 0x19, 0xb6,
0xd2, 0x8f, 0x62, 0x81, 0x49, 0x1a, 0xde, 0xe4, 0xdf, 0x3f, 0xf8, 0x0d, 0x00, 0x00, 0xff, 0xff,
0x9f, 0x12, 0xb7, 0x99, 0xa6, 0x01, 0x00, 0x00,
var fileDescriptor_resource_reference_05b185b0f444334d = []byte{
// 325 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x41, 0x6b, 0xc2, 0x40,
0x10, 0x85, 0x4d, 0x62, 0x2b, 0x8e, 0x22, 0x71, 0x69, 0x21, 0xbd, 0x89, 0xb4, 0x20, 0x1e, 0x12,
0x50, 0xbc, 0x57, 0xed, 0x42, 0xad, 0x6d, 0x22, 0x6b, 0xc4, 0xb6, 0x97, 0x90, 0xc4, 0x51, 0x17,
0x63, 0x76, 0x89, 0x91, 0x92, 0x6b, 0x7f, 0x79, 0x31, 0x54, 0xa2, 0xb7, 0x81, 0xef, 0xf1, 0xcd,
0x83, 0x07, 0x8f, 0x81, 0x1f, 0xee, 0x30, 0x5e, 0x59, 0xbe, 0xe4, 0x56, 0x82, 0x07, 0x71, 0x4c,
0x42, 0xf4, 0x12, 0x5c, 0x63, 0x82, 0x71, 0x88, 0xa6, 0x4c, 0x44, 0x2a, 0x88, 0xe6, 0x4b, 0xde,
0x7e, 0x81, 0x1a, 0xfb, 0x0f, 0x4c, 0x31, 0x23, 0x4f, 0x50, 0x4e, 0x33, 0x89, 0x86, 0xd2, 0x52,
0x3a, 0x8d, 0x5e, 0xd3, 0xf4, 0x25, 0x37, 0xcf, 0xdc, 0xcd, 0x24, 0xb2, 0x1c, 0x93, 0x06, 0xa8,
0x7c, 0x65, 0xa8, 0x2d, 0xa5, 0x53, 0x65, 0x2a, 0x5f, 0xb5, 0x7f, 0x15, 0x68, 0x9e, 0x63, 0xec,
0xfc, 0x86, 0xb4, 0x41, 0xdb, 0x61, 0x96, 0xbb, 0x6a, 0x3d, 0xfd, 0xca, 0x35, 0xc5, 0x8c, 0x9d,
0x20, 0x21, 0x50, 0x8e, 0xfd, 0x3d, 0x1a, 0x5a, 0xee, 0xca, 0x6f, 0x32, 0x80, 0x7a, 0x82, 0x91,
0x9f, 0x72, 0x11, 0x1f, 0xb6, 0x5c, 0xe6, 0x7f, 0x8a, 0x32, 0x05, 0x60, 0x57, 0xb1, 0xee, 0x08,
0xea, 0x97, 0x55, 0xc9, 0x03, 0xdc, 0x2f, 0xec, 0xa9, 0xed, 0x2c, 0x6d, 0x8f, 0xd1, 0xb9, 0xb3,
0x60, 0x63, 0xea, 0xb9, 0x5f, 0x33, 0xaa, 0x97, 0x48, 0x03, 0x80, 0x7e, 0xce, 0x28, 0x9b, 0x7c,
0x50, 0xdb, 0xd5, 0x15, 0x52, 0x01, 0xed, 0xcd, 0x19, 0xe9, 0x6a, 0xf7, 0xf9, 0xe4, 0x28, 0x9c,
0xc4, 0x80, 0xbb, 0xc2, 0xf1, 0x3e, 0x74, 0x27, 0x8e, 0x3d, 0x7f, 0x9d, 0xcc, 0xf4, 0x12, 0xa9,
0xc2, 0x8d, 0xb3, 0xb4, 0x29, 0xd3, 0x15, 0x52, 0x83, 0xca, 0x98, 0xd1, 0xa1, 0xeb, 0x30, 0x5d,
0x1d, 0x0d, 0xbe, 0xfb, 0x1b, 0x9e, 0x6e, 0x8f, 0x81, 0x19, 0x8a, 0xbd, 0xb5, 0x3b, 0x06, 0xb8,
0x8e, 0xc4, 0x8f, 0x25, 0xb9, 0xc4, 0x88, 0xc7, 0x78, 0xb0, 0x2e, 0xb7, 0xd9, 0x08, 0x2f, 0x8c,
0x38, 0xc6, 0x69, 0x70, 0x9b, 0x6f, 0xd2, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xa6, 0xdd, 0xe5,
0xee, 0xbb, 0x01, 0x00, 0x00,
}

3
backend/api/go_http_client/job_model/api_resource_reference.go
View File

@ -33,6 +33,9 @@ type APIResourceReference struct {
// key
Key *APIResourceKey `json:"key,omitempty"`
// The name of the resource that referred to.
Name string `json:"name,omitempty"`
// Required field. The relationship from referred resource to the object.
Relationship APIRelationship `json:"relationship,omitempty"`
}

3
backend/api/go_http_client/run_model/api_resource_reference.go
View File

@ -33,6 +33,9 @@ type APIResourceReference struct {
// key
Key *APIResourceKey `json:"key,omitempty"`
// The name of the resource that referred to.
Name string `json:"name,omitempty"`
// Required field. The relationship from referred resource to the object.
Relationship APIRelationship `json:"relationship,omitempty"`
}

3
backend/api/resource_reference.proto
View File

@ -41,6 +41,9 @@ message ResourceKey {
message ResourceReference {
ResourceKey key = 1;
// The name of the resource that referred to.
string name = 3;
// Required field. The relationship from referred resource to the object.
Relationship relationship = 2;
}

4
backend/api/swagger/job.swagger.json generated
View File

@ -437,6 +437,10 @@
"key": {
"$ref": "#/definitions/apiResourceKey"
},
"name": {
"type": "string",
"description": "The name of the resource that referred to."
},
"relationship": {
"$ref": "#/definitions/apiRelationship",
"description": "Required field. The relationship from referred resource to the object."

4
backend/api/swagger/kfp_api_single_file.swagger.json generated
View File

@ -1144,6 +1144,10 @@
"key": {
"$ref": "#/definitions/apiResourceKey"
},
"name": {
"type": "string",
"description": "The name of the resource that referred to."
},
"relationship": {
"$ref": "#/definitions/apiRelationship",
"description": "Required field. The relationship from referred resource to the object."

4
backend/api/swagger/run.swagger.json generated
View File

@ -565,6 +565,10 @@
"key": {
"$ref": "#/definitions/apiResourceKey"
},
"name": {
"type": "string",
"description": "The name of the resource that referred to."
},
"relationship": {
"$ref": "#/definitions/apiRelationship",
"description": "Required field. The relationship from referred resource to the object."

4
backend/src/apiserver/model/experiment.go
View File

@ -7,8 +7,8 @@ type Experiment struct {
CreatedAtInSec int64 `gorm:"column:CreatedAtInSec; not null"`
}
func (r Experiment) GetValueOfPrimaryKey() string {
return r.UUID
func (e Experiment) GetValueOfPrimaryKey() string {
return e.UUID
}
func GetExperimentTablePrimaryKeyColumn() string {

5
backend/src/apiserver/model/resource_reference.go
View File

@ -27,6 +27,9 @@ type ResourceReference struct {
// The ID of the resource that been referenced to.
ReferenceUUID string `gorm:"column:ReferenceUUID; not null; "`
// The name of the resource that been referenced to.
ReferenceName string `gorm:"column:ReferenceName; not null; "`
// The type of the resource that been referenced to.
ReferenceType common.ResourceType `gorm:"column:ReferenceType; not null; primary_key"`
@ -34,5 +37,5 @@ type ResourceReference struct {
Relationship common.Relationship `gorm:"column:Relationship; not null; "`
// The json formatted blob of the resource reference.
Payload string `gorm:"column:Payload; not null; "`
Payload string `gorm:"column:Payload; not null; size:65535 "`
}

50
backend/src/apiserver/resource/model_converter.go
View File

@ -24,7 +24,7 @@ import (
"github.com/kubeflow/pipelines/backend/src/common/util"
)
func ToModelRunMetric(metric *api.RunMetric, runUUID string) *model.RunMetric {
func (r *ResourceManager) ToModelRunMetric(metric *api.RunMetric, runUUID string) *model.RunMetric {
return &model.RunMetric{
RunUUID: runUUID,
Name: metric.GetName(),
@ -36,12 +36,12 @@ func ToModelRunMetric(metric *api.RunMetric, runUUID string) *model.RunMetric {
// The input run might not contain workflowSpecManifest, but instead a pipeline ID.
// The caller would retrieve workflowSpecManifest and pass in.
func ToModelRunDetail(run *api.Run, runId string, workflow *util.Workflow, workflowSpecManifest string) (*model.RunDetail, error) {
func (r *ResourceManager) ToModelRunDetail(run *api.Run, runId string, workflow *util.Workflow, workflowSpecManifest string) (*model.RunDetail, error) {
params, err := toModelParameters(run.PipelineSpec.Parameters)
if err != nil {
return nil, util.Wrap(err, "Unable to parse the parameter.")
}
resourceReferences, err := toModelResourceReferences(runId, common.Run, run.ResourceReferences)
resourceReferences, err := r.toModelResourceReferences(runId, common.Run, run.ResourceReferences)
if err != nil {
return nil, util.Wrap(err, "Unable to convert resource references.")
}
@ -67,12 +67,12 @@ func ToModelRunDetail(run *api.Run, runId string, workflow *util.Workflow, workf
}, nil
}
func ToModelJob(job *api.Job, swf *util.ScheduledWorkflow, workflowSpecManifest string) (*model.Job, error) {
func (r *ResourceManager) ToModelJob(job *api.Job, swf *util.ScheduledWorkflow, workflowSpecManifest string) (*model.Job, error) {
params, err := toModelParameters(job.PipelineSpec.Parameters)
if err != nil {
return nil, util.Wrap(err, "Error parsing the input job.")
}
resourceReferences, err := toModelResourceReferences(string(swf.UID), common.Job, job.ResourceReferences)
resourceReferences, err := r.toModelResourceReferences(string(swf.UID), common.Job, job.ResourceReferences)
if err != nil {
return nil, util.Wrap(err, "Error to convert resource references.")
}
@ -145,8 +145,8 @@ func toModelParameters(apiParams []*api.Parameter) (string, error) {
return string(paramsBytes), nil
}
func toModelResourceReferences(
resourceId string, resourceType common.ResourceType, apiRefs []*api.ResourceReference) ([]*model.ResourceReference, error) {
func (r *ResourceManager) toModelResourceReferences(
resourceId string, resourceType common.ResourceType, apiRefs []*api.ResourceReference) ([]*model.ResourceReference, error) {
var modelRefs []*model.ResourceReference
for _, apiRef := range apiRefs {
modelReferenceType, err := common.ToModelResourceType(apiRef.Key.Type)
@ -157,10 +157,15 @@ func toModelResourceReferences(
if err != nil {
return nil, util.Wrap(err, "Failed to convert relationship")
}
referenceName, err := r.getResourceName(modelReferenceType, apiRef.Key.Id)
if err != nil {
return nil, util.Wrap(err, "Failed to find the referred resource")
}
modelRef := &model.ResourceReference{
ResourceUUID: resourceId,
ResourceType: resourceType,
ReferenceUUID: apiRef.Key.Id,
ReferenceName: referenceName,
ReferenceType: modelReferenceType,
Relationship: modelRelationship,
}
@ -168,3 +173,34 @@ func toModelResourceReferences(
}
return modelRefs, nil
}
func (r *ResourceManager) getResourceName(resourceType common.ResourceType, resourceId string) (string, error) {
switch resourceType {
case common.Experiment:
experiment, err := r.GetExperiment(resourceId)
if err != nil {
return "", util.Wrap(err, "Referred experiment not found.")
}
return experiment.Name, nil
case common.Pipeline:
pipeline, err := r.GetPipeline(resourceId)
if err != nil {
return "", util.Wrap(err, "Referred pipeline not found.")
}
return pipeline.Name, nil
case common.Job:
job, err := r.GetJob(resourceId)
if err != nil {
return "", util.Wrap(err, "Referred job not found.")
}
return job.DisplayName, nil
case common.Run:
run, err := r.GetRun(resourceId)
if err != nil {
return "", util.Wrap(err, "Referred run not found.")
}
return run.DisplayName, nil
default:
return "", util.NewInvalidInputError("Unsupported resource type: %s", string(resourceType))
}
}

71
backend/src/apiserver/resource/model_converter_test.go
View File

@ -28,7 +28,14 @@ import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func initResourceManager() (*FakeClientManager, *ResourceManager) {
store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch())
return store, NewResourceManager(store)
}
func TestToModelRunMetric(t *testing.T) {
store, manager := initResourceManager()
defer store.Close()
apiRunMetric := &api.RunMetric{
Name: "metric-1",
NodeId: "node-1",
@ -38,7 +45,7 @@ func TestToModelRunMetric(t *testing.T) {
Format: api.RunMetric_RAW,
}
actualModelRunMetric := ToModelRunMetric(apiRunMetric, "run-1")
actualModelRunMetric := manager.ToModelRunMetric(apiRunMetric, "run-1")
expectedModelRunMetric := &model.RunMetric{
RunUUID: "run-1",
@ -51,6 +58,9 @@ func TestToModelRunMetric(t *testing.T) {
}
func TestToModelRunDetail(t *testing.T) {
store, manager, experiment := initWithExperiment(t)
defer store.Close()
apiRun := &api.Run{
Id: "run1",
Name: "name1",
@ -59,14 +69,14 @@ func TestToModelRunDetail(t *testing.T) {
Parameters: []*api.Parameter{{Name: "param2", Value: "world"}},
},
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "exp1"}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID}, Relationship: api.Relationship_OWNER},
},
}
workflow := util.NewWorkflow(&v1alpha1.Workflow{
ObjectMeta: v1.ObjectMeta{Name: "workflow-name", UID: "123"},
Status: v1alpha1.WorkflowStatus{Phase: "running"},
})
modelRunDetail, err := ToModelRunDetail(apiRun, "123", workflow, "workflow spec")
modelRunDetail, err := manager.ToModelRunDetail(apiRun, "123", workflow, "workflow spec")
assert.Nil(t, err)
expectedModelRunDetail := &model.RunDetail{
@ -84,7 +94,8 @@ func TestToModelRunDetail(t *testing.T) {
{
ResourceUUID: "123",
ResourceType: common.Run,
ReferenceUUID: "exp1",
ReferenceUUID: experiment.UUID,
ReferenceName: experiment.Name,
ReferenceType: common.Experiment,
Relationship: common.Owner},
},
@ -97,6 +108,8 @@ func TestToModelRunDetail(t *testing.T) {
}
func TestToModelJob(t *testing.T) {
store, manager, experiment := initWithExperiment(t)
defer store.Close()
apiJob := &api.Job{
Name: "name1",
Enabled: true,
@ -107,7 +120,7 @@ func TestToModelJob(t *testing.T) {
Cron: "1 * * * *",
}}},
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "exp1"}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID}, Relationship: api.Relationship_OWNER},
},
PipelineSpec: &api.PipelineSpec{PipelineId: "1", Parameters: []*api.Parameter{{Name: "param2", Value: "world"}}},
}
@ -120,7 +133,7 @@ func TestToModelJob(t *testing.T) {
Status: swfapi.ScheduledWorkflowStatus{
Conditions: []swfapi.ScheduledWorkflowCondition{{Type: swfapi.ScheduledWorkflowEnabled}}},
})
modelJob, err := ToModelJob(apiJob, swf, "workflow spec")
modelJob, err := manager.ToModelJob(apiJob, swf, "workflow spec")
assert.Nil(t, err)
expectedModelJob := &model.Job{
@ -146,7 +159,8 @@ func TestToModelJob(t *testing.T) {
{
ResourceUUID: "swf_123",
ResourceType: common.Job,
ReferenceUUID: "exp1",
ReferenceUUID: experiment.UUID,
ReferenceName: experiment.Name,
ReferenceType: common.Experiment,
Relationship: common.Owner},
},
@ -155,22 +169,27 @@ func TestToModelJob(t *testing.T) {
}
func TestToModelResourceReferences(t *testing.T) {
refs, err := toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "e1"}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "j1"}, Relationship: api.Relationship_CREATOR},
store, manager, job := initWithJob(t)
defer store.Close()
refs, err := manager.toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: DefaultFakeUUID}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: job.UUID}, Relationship: api.Relationship_CREATOR},
})
assert.Nil(t, err)
expectedRefs := []*model.ResourceReference{
{ResourceUUID: "r1", ResourceType: common.Run,
ReferenceUUID: "e1", ReferenceType: common.Experiment, Relationship: common.Owner},
ReferenceUUID: DefaultFakeUUID, ReferenceName: "e1", ReferenceType: common.Experiment, Relationship: common.Owner},
{ResourceUUID: "r1", ResourceType: common.Run,
ReferenceUUID: "j1", ReferenceType: common.Job, Relationship: common.Creator},
ReferenceUUID: job.UUID, ReferenceName: "j1", ReferenceType: common.Job, Relationship: common.Creator},
}
assert.Equal(t, expectedRefs, refs)
}
func TestToModelResourceReferences_UnknownRefType(t *testing.T) {
_, err := toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
store, manager, _ := initWithJob(t)
defer store.Close()
_, err := manager.toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_UNKNOWN_RESOURCE_TYPE, Id: "e1"}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "j1"}, Relationship: api.Relationship_CREATOR},
})
@ -179,10 +198,34 @@ func TestToModelResourceReferences_UnknownRefType(t *testing.T) {
}
func TestToModelResourceReferences_UnknownRelationship(t *testing.T) {
_, err := toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
store, manager, _ := initWithJob(t)
defer store.Close()
_, err := manager.toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "e1"}, Relationship: api.Relationship_UNKNOWN_RELATIONSHIP},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "j1"}, Relationship: api.Relationship_CREATOR},
})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to convert relationship")
}
func TestToModelResourceReferences_ReferredJobNotFound(t *testing.T) {
store, manager, _ := initWithJob(t)
defer store.Close()
_, err := manager.toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "e1"}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "j2"}, Relationship: api.Relationship_CREATOR},
})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to find the referred resource")
}
func TestToModelResourceReferences_ReferredExperimentNotFound(t *testing.T) {
store, manager, _ := initWithJob(t)
defer store.Close()
_, err := manager.toModelResourceReferences("r1", common.Run, []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "e2"}, Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "j1"}, Relationship: api.Relationship_CREATOR},
})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to find the referred resource")
}

13
backend/src/apiserver/resource/resource_manager.go
View File

@ -256,7 +256,7 @@ func (r *ResourceManager) CreateRun(apiRun *api.Run) (*model.RunDetail, error) {
}
// Store run metadata into database
runDetail, err := ToModelRunDetail(apiRun, runId, util.NewWorkflow(newWorkflow), string(workflowSpecManifestBytes))
runDetail, err := r.ToModelRunDetail(apiRun, runId, util.NewWorkflow(newWorkflow), string(workflowSpecManifestBytes))
if err != nil {
return nil, util.Wrap(err, "Failed to convert run model")
}
@ -439,7 +439,7 @@ func (r *ResourceManager) CreateJob(apiJob *api.Job) (*model.Job, error) {
apiJob.ResourceReferences = append(apiJob.ResourceReferences, ref)
}
job, err := ToModelJob(apiJob, util.NewScheduledWorkflow(newScheduledWorkflow), string(workflowSpecManifestBytes))
job, err := r.ToModelJob(apiJob, util.NewScheduledWorkflow(newScheduledWorkflow), string(workflowSpecManifestBytes))
if err != nil {
return nil, util.Wrap(err, "Create job failed")
}
@ -509,6 +509,7 @@ func (r *ResourceManager) ReportWorkflowResource(workflow *util.Workflow) error
if jobId == "" {
// If a run doesn't have job ID, it's a one-time run created by Pipeline API server.
// In this case the DB entry should already been created when argo workflow CRD is created.
err := r.runStore.UpdateRun(runId, workflow.Condition(), workflow.FinishedAt(), workflow.ToStringForStore())
if err != nil {
return util.Wrap(err, "Failed to update the run.")
@ -519,6 +520,10 @@ func (r *ResourceManager) ReportWorkflowResource(workflow *util.Workflow) error
if err != nil {
return util.Wrap(err, "Failed to retrieve the experiment ID for the job that created the run.")
}
jobName, err := r.getResourceName(common.Job, jobId)
if err != nil {
return util.Wrap(err, "Failed to retrieve the job name for the job that created the run.")
}
runDetail := &model.RunDetail{
Run: model.Run{
UUID: runId,
@ -538,6 +543,7 @@ func (r *ResourceManager) ReportWorkflowResource(workflow *util.Workflow) error
ResourceUUID: runId,
ResourceType: common.Run,
ReferenceUUID: jobId,
ReferenceName: jobName,
ReferenceType: common.Job,
Relationship: common.Creator,
},
@ -545,6 +551,7 @@ func (r *ResourceManager) ReportWorkflowResource(workflow *util.Workflow) error
ResourceUUID: runId,
ResourceType: common.Run,
ReferenceUUID: experimentRef.ReferenceUUID,
ReferenceName: experimentRef.ReferenceName,
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -709,7 +716,7 @@ func (r *ResourceManager) getDefaultExperimentIfNoExperiment(references []*api.R
}
func (r *ResourceManager) ReportMetric(metric *api.RunMetric, runUUID string) error {
return r.runStore.ReportMetric(ToModelRunMetric(metric, runUUID))
return r.runStore.ReportMetric(r.ToModelRunMetric(metric, runUUID))
}
// ReadArtifact parses run's workflow to find artifact file path and reads the content of the file

13
backend/src/apiserver/resource/resource_manager_test.go
View File

@ -99,6 +99,7 @@ func initWithJob(t *testing.T) (*FakeClientManager, *ResourceManager, *model.Job
}
j, err := manager.CreateJob(job)
assert.Nil(t, err)
return store, manager, j
}
@ -290,6 +291,7 @@ func TestCreateRun_ThroughPipelineID(t *testing.T) {
ResourceUUID: "123e4567-e89b-12d3-a456-426655440000",
ResourceType: common.Run,
ReferenceUUID: experiment.UUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -329,6 +331,7 @@ func TestCreateRun_ThroughWorkflowSpec(t *testing.T) {
ResourceUUID: "123e4567-e89b-12d3-a456-426655440000",
ResourceType: common.Run,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -366,6 +369,7 @@ func TestCreateRun_NoExperiment(t *testing.T) {
ResourceType: common.Run,
// Experiment is now set
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "Default",
ReferenceType: common.Experiment,
Relationship: common.Owner,
}}
@ -667,6 +671,7 @@ func TestCreateJob_ThroughWorkflowSpec(t *testing.T) {
ResourceUUID: "123",
ResourceType: common.Job,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -716,6 +721,7 @@ func TestCreateJob_ThroughPipelineID(t *testing.T) {
ResourceUUID: "123",
ResourceType: common.Job,
ReferenceUUID: experiment.UUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -816,6 +822,7 @@ func TestEnableJob(t *testing.T) {
ResourceUUID: "123",
ResourceType: common.Job,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -923,6 +930,7 @@ func TestReportWorkflowResource_ScheduledWorkflowIDEmpty_Success(t *testing.T) {
ResourceUUID: "123e4567-e89b-12d3-a456-426655440000",
ResourceType: common.Run,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -975,6 +983,7 @@ func TestReportWorkflowResource_ScheduledWorkflowIDNotEmpty_Success(t *testing.T
ResourceUUID: "WORKFLOW_1",
ResourceType: common.Run,
ReferenceUUID: job.UUID,
ReferenceName: job.Name,
ReferenceType: common.Job,
Relationship: common.Creator,
},
@ -982,6 +991,7 @@ func TestReportWorkflowResource_ScheduledWorkflowIDNotEmpty_Success(t *testing.T
ResourceUUID: "WORKFLOW_1",
ResourceType: common.Run,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -1046,6 +1056,7 @@ func TestReportWorkflowResource_ScheduledWorkflowIDNotEmpty_NoExperiment_Success
ResourceUUID: "WORKFLOW_1",
ResourceType: common.Run,
ReferenceUUID: newJob.UUID,
ReferenceName: newJob.Name,
ReferenceType: common.Job,
Relationship: common.Creator,
},
@ -1053,6 +1064,7 @@ func TestReportWorkflowResource_ScheduledWorkflowIDNotEmpty_NoExperiment_Success
ResourceUUID: "WORKFLOW_1",
ResourceType: common.Run,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "Default",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -1159,6 +1171,7 @@ func TestReportScheduledWorkflowResource_Success(t *testing.T) {
ResourceUUID: job.UUID,
ResourceType: common.Job,
ReferenceUUID: DefaultFakeUUID,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},

1
backend/src/apiserver/server/api_converter.go
View File

@ -205,6 +205,7 @@ func toApiResourceReferences(references []*model.ResourceReference) []*api.Resou
Type: toApiResourceType(ref.ReferenceType),
Id: ref.ReferenceUUID,
},
Name: ref.ReferenceName,
Relationship: toApiRelationship(ref.Relationship),
})
}

30
backend/src/apiserver/server/api_converter_test.go
View File

@ -70,8 +70,8 @@ func TestToApiRunDetail(t *testing.T) {
WorkflowSpecManifest: "manifest",
},
ResourceReferences: []*model.ResourceReference{
{ResourceUUID: "run123", ResourceType: common.Run,
ReferenceUUID: "job123", ReferenceType: common.Job, Relationship: common.Creator},
{ResourceUUID: "run123", ResourceType: common.Run, ReferenceUUID: "job123",
ReferenceName: "j123", ReferenceType: common.Job, Relationship: common.Creator},
},
},
PipelineRuntime: model.PipelineRuntime{WorkflowRuntimeManifest: "workflow123"},
@ -91,7 +91,7 @@ func TestToApiRunDetail(t *testing.T) {
},
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "job123"},
Relationship: api.Relationship_CREATOR},
Name: "j123", Relationship: api.Relationship_CREATOR},
},
},
PipelineRuntime: &api.PipelineRuntime{
@ -139,8 +139,8 @@ func TestToApiRuns(t *testing.T) {
WorkflowSpecManifest: "manifest",
},
ResourceReferences: []*model.ResourceReference{
{ResourceUUID: "run1", ResourceType: common.Run,
ReferenceUUID: "job1", ReferenceType: common.Job, Relationship: common.Creator},
{ResourceUUID: "run1", ResourceType: common.Run,ReferenceUUID: "job1",
ReferenceName:"j1",ReferenceType: common.Job, Relationship: common.Creator},
},
Metrics: []*model.RunMetric{metric1, metric2},
}
@ -157,8 +157,8 @@ func TestToApiRuns(t *testing.T) {
WorkflowSpecManifest: "manifest",
},
ResourceReferences: []*model.ResourceReference{
{ResourceUUID: "run2", ResourceType: common.Run,
ReferenceUUID: "job2", ReferenceType: common.Job, Relationship: common.Creator},
{ResourceUUID: "run2", ResourceType: common.Run,ReferenceUUID: "job2",
ReferenceName:"j2", ReferenceType: common.Job, Relationship: common.Creator},
},
Metrics: []*model.RunMetric{metric2},
}
@ -177,7 +177,7 @@ func TestToApiRuns(t *testing.T) {
},
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "job1"},
Relationship: api.Relationship_CREATOR},
Name:"j1", Relationship: api.Relationship_CREATOR},
},
Metrics: []*api.RunMetric{apiMetric1, apiMetric2},
},
@ -191,7 +191,7 @@ func TestToApiRuns(t *testing.T) {
Status: "done",
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "job2"},
Relationship: api.Relationship_CREATOR},
Name:"j2", Relationship: api.Relationship_CREATOR},
},
PipelineSpec: &api.PipelineSpec{
WorkflowManifest: "manifest",
@ -222,7 +222,7 @@ func TestCronScheduledJobToApiJob(t *testing.T) {
CreatedAtInSec: 1,
UpdatedAtInSec: 1,
ResourceReferences: []*model.ResourceReference{
{ResourceUUID: "job1", ResourceType: common.Job, ReferenceUUID: "experiment1",
{ResourceUUID: "job1", ResourceType: common.Job, ReferenceUUID: "experiment1", ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Owner},
},
}
@ -245,7 +245,7 @@ func TestCronScheduledJobToApiJob(t *testing.T) {
},
ResourceReferences: []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "experiment1"},
Relationship: api.Relationship_OWNER},
Name: "e1", Relationship: api.Relationship_OWNER},
},
}
assert.Equal(t, expectedJob, apiJob)
@ -472,15 +472,15 @@ func TestToApiRunMetric_UnknownFormat(t *testing.T) {
func TestToApiResourceReferences(t *testing.T) {
resourceReferences := []*model.ResourceReference{
{ResourceUUID: "run1", ResourceType: common.Run, ReferenceUUID: "experiment1",
ReferenceType: common.Experiment, Relationship: common.Owner},
ReferenceName: "e1", ReferenceType: common.Experiment, Relationship: common.Owner},
{ResourceUUID: "run1", ResourceType: common.Run, ReferenceUUID: "job1",
ReferenceType: common.Job, Relationship: common.Owner},
ReferenceName: "j1", ReferenceType: common.Job, Relationship: common.Owner},
}
expectedApiResourceReferences := []*api.ResourceReference{
{Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: "experiment1"},
Relationship: api.Relationship_OWNER},
Name: "e1", Relationship: api.Relationship_OWNER},
{Key: &api.ResourceKey{Type: api.ResourceType_JOB, Id: "job1"},
Relationship: api.Relationship_OWNER},
Name: "j1", Relationship: api.Relationship_OWNER},
}
assert.Equal(t, expectedApiResourceReferences, toApiResourceReferences(resourceReferences))
}

8
backend/src/apiserver/server/run_server_test.go
View File

@ -45,8 +45,8 @@ func TestCreateRun(t *testing.T) {
},
ResourceReferences: []*api.ResourceReference{
{
Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID},
Relationship: api.Relationship_OWNER,
Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID},
Name: "123", Relationship: api.Relationship_OWNER,
},
},
},
@ -85,8 +85,8 @@ func TestListRun(t *testing.T) {
},
ResourceReferences: []*api.ResourceReference{
{
Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID},
Relationship: api.Relationship_OWNER,
Key: &api.ResourceKey{Type: api.ResourceType_EXPERIMENT, Id: experiment.UUID},
Name: "123", Relationship: api.Relationship_OWNER,
},
},
}

82
backend/src/apiserver/storage/job_store_test.go
View File

@ -63,8 +63,8 @@ func initializeDbAndStore() (*DB, *JobStore) {
UpdatedAtInSec: 1,
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
ResourceUUID: "1", ResourceType: common.Job, ReferenceUUID: defaultFakeExpId,
ReferenceName: "e1", ReferenceType: common.Experiment,
Relationship: common.Owner,
},
},
@ -83,7 +83,7 @@ func initializeDbAndStore() (*DB, *JobStore) {
CronSchedule: model.CronSchedule{
CronScheduleStartTimeInSec: util.Int64Pointer(1),
CronScheduleEndTimeInSec: util.Int64Pointer(2),
Cron: util.StringPointer("1 * *"),
Cron: util.StringPointer("1 * *"),
},
},
Enabled: true,
@ -92,7 +92,7 @@ func initializeDbAndStore() (*DB, *JobStore) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName: "e2", ReferenceType: common.Experiment,
Relationship: common.Owner,
},
},
@ -128,8 +128,8 @@ func TestListJobs_Pagination(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}}
@ -156,7 +156,7 @@ func TestListJobs_Pagination(t *testing.T) {
CronSchedule: model.CronSchedule{
CronScheduleStartTimeInSec: util.Int64Pointer(1),
CronScheduleEndTimeInSec: util.Int64Pointer(2),
Cron: util.StringPointer("1 * *"),
Cron: util.StringPointer("1 * *"),
},
},
CreatedAtInSec: 2,
@ -165,8 +165,8 @@ func TestListJobs_Pagination(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName:"e2",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}}
@ -236,7 +236,7 @@ func TestListJobs_Pagination_Descent(t *testing.T) {
CronSchedule: model.CronSchedule{
CronScheduleStartTimeInSec: util.Int64Pointer(1),
CronScheduleEndTimeInSec: util.Int64Pointer(2),
Cron: util.StringPointer("1 * *"),
Cron: util.StringPointer("1 * *"),
},
},
CreatedAtInSec: 2,
@ -244,8 +244,8 @@ func TestListJobs_Pagination_Descent(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName:"e2",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}}
@ -280,8 +280,8 @@ func TestListJobs_Pagination_Descent(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}}
@ -322,8 +322,8 @@ func TestListJobs_Pagination_LessThanPageSize(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
},
@ -341,7 +341,7 @@ func TestListJobs_Pagination_LessThanPageSize(t *testing.T) {
CronSchedule: model.CronSchedule{
CronScheduleStartTimeInSec: util.Int64Pointer(1),
CronScheduleEndTimeInSec: util.Int64Pointer(2),
Cron: util.StringPointer("1 * *"),
Cron: util.StringPointer("1 * *"),
},
},
CreatedAtInSec: 2,
@ -349,8 +349,8 @@ func TestListJobs_Pagination_LessThanPageSize(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName:"e2",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}}
@ -391,8 +391,8 @@ func TestListJobs_FilterByReferenceKey(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}}
@ -446,8 +446,8 @@ func TestGetJob(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -496,7 +496,7 @@ func TestCreateJob(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1", ReferenceType: common.Experiment,
Relationship: common.Owner,
},
},
@ -518,7 +518,7 @@ func TestCreateJob(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1", ReferenceType: common.Experiment,
Relationship: common.Owner,
},
}}
@ -548,8 +548,8 @@ func TestCreateJobError(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -588,8 +588,8 @@ func TestEnableJob(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -628,8 +628,8 @@ func TestEnableJob_SkipUpdate(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -676,8 +676,8 @@ func TestUpdateJob_Success(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId,ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -747,7 +747,7 @@ func TestUpdateJob_Success(t *testing.T) {
CronSchedule: model.CronSchedule{
CronScheduleStartTimeInSec: util.Int64Pointer(10),
CronScheduleEndTimeInSec: util.Int64Pointer(20),
Cron: util.StringPointer("MY_CRON"),
Cron: util.StringPointer("MY_CRON"),
},
PeriodicSchedule: model.PeriodicSchedule{
PeriodicScheduleStartTimeInSec: util.Int64Pointer(30),
@ -758,8 +758,8 @@ func TestUpdateJob_Success(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -795,8 +795,8 @@ func TestUpdateJob_MostlyEmptySpec(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}
@ -833,7 +833,7 @@ func TestUpdateJob_MostlyEmptySpec(t *testing.T) {
CronSchedule: model.CronSchedule{
CronScheduleStartTimeInSec: nil,
CronScheduleEndTimeInSec: nil,
Cron: util.StringPointer(""),
Cron: util.StringPointer(""),
},
PeriodicSchedule: model.PeriodicSchedule{
PeriodicScheduleStartTimeInSec: nil,
@ -844,8 +844,8 @@ func TestUpdateJob_MostlyEmptySpec(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpId, ReferenceName:"e1",
ReferenceType: common.Experiment, Relationship: common.Owner,
},
},
}

38
backend/src/apiserver/storage/resource_reference_store.go
View File

@ -12,10 +12,13 @@ import (
"k8s.io/apimachinery/pkg/util/json"
)
var resourceReferenceColumns = []string{"ResourceUUID", "ResourceType", "ReferenceUUID",
"ReferenceName", "ReferenceType", "Relationship", "Payload"}
type ResourceReferenceStoreInterface interface {
// Retrieve the resource reference for a given resource id, type and a reference type.
GetResourceReference(resourceId string, resourceType common.ResourceType,
referenceType common.ResourceType) (*model.ResourceReference, error)
referenceType common.ResourceType) (*model.ResourceReference, error)
}
type ResourceReferenceStore struct {
@ -28,7 +31,7 @@ func (s *ResourceReferenceStore) CreateResourceReferences(tx *sql.Tx, refs []*mo
if len(refs) > 0 {
resourceRefSqlBuilder := sq.
Insert("resource_references").
Columns("ResourceUUID", "ResourceType", "ReferenceUUID", "ReferenceType", "Relationship", "Payload")
Columns("ResourceUUID", "ResourceType", "ReferenceUUID", "ReferenceName", "ReferenceType", "Relationship", "Payload")
for _, ref := range refs {
if !s.checkReferenceExist(tx, ref.ReferenceUUID, ref.ReferenceType) {
return util.NewResourceNotFoundError(string(ref.ReferenceType), ref.ReferenceUUID)
@ -38,7 +41,7 @@ func (s *ResourceReferenceStore) CreateResourceReferences(tx *sql.Tx, refs []*mo
return util.NewInternalServerError(err, "Failed to stream resource reference model to a json payload")
}
resourceRefSqlBuilder = resourceRefSqlBuilder.Values(
ref.ResourceUUID, ref.ResourceType, ref.ReferenceUUID, ref.ReferenceType, ref.Relationship, string(payload))
ref.ResourceUUID, ref.ResourceType, ref.ReferenceUUID, ref.ReferenceName, ref.ReferenceType, ref.Relationship, string(payload))
}
refSql, refArgs, err := resourceRefSqlBuilder.ToSql()
if err != nil {
@ -79,9 +82,9 @@ func (s *ResourceReferenceStore) checkReferenceExist(tx *sql.Tx, referenceId str
func (s *ResourceReferenceStore) DeleteResourceReferences(tx *sql.Tx, id string, resourceType common.ResourceType) error {
refSql, refArgs, err := sq.
Delete("resource_references").
Where(sq.Or{
sq.Eq{"ResourceUUID": id, "ResourceType": resourceType},
sq.Eq{"ReferenceUUID": id, "ReferenceType": resourceType}}).
Where(sq.Or{
sq.Eq{"ResourceUUID": id, "ResourceType": resourceType},
sq.Eq{"ReferenceUUID": id, "ReferenceType": resourceType}}).
ToSql()
_, err = tx.Exec(refSql, refArgs...)
if err != nil {
@ -91,29 +94,29 @@ func (s *ResourceReferenceStore) DeleteResourceReferences(tx *sql.Tx, id string,
}
func (s *ResourceReferenceStore) GetResourceReference(resourceId string, resourceType common.ResourceType,
referenceType common.ResourceType) (*model.ResourceReference, error) {
sql, args, err := sq.Select("*").
referenceType common.ResourceType) (*model.ResourceReference, error) {
sql, args, err := sq.Select(resourceReferenceColumns...).
From("resource_references").
Where(sq.Eq{
"ResourceUUID": resourceId,
"ResourceType": resourceType,
"ReferenceType": referenceType}).
Where(sq.Eq{
"ResourceUUID": resourceId,
"ResourceType": resourceType,
"ReferenceType": referenceType}).
Limit(1).ToSql()
if err != nil {
return nil, util.NewInternalServerError(err,
"Failed to create query to get resource reference. "+
"Resource ID: %s. Resource Type: %s. Reference Type: %s", resourceId, resourceType, referenceType)
"Resource ID: %s. Resource Type: %s. Reference Type: %s", resourceId, resourceType, referenceType)
}
row, err := s.db.Query(sql, args...)
if err != nil {
return nil, util.NewInternalServerError(err,
"Failed to get resource reference. "+
"Resource ID: %s. Resource Type: %s. Reference Type: %s", resourceId, resourceType, referenceType)
"Resource ID: %s. Resource Type: %s. Reference Type: %s", resourceId, resourceType, referenceType)
}
defer row.Close()
reference, err := s.scanRows(row)
if err != nil || len(reference) > 1 {
return nil, util.NewInternalServerError(err, "Failed to get job: %v", err.Error())
return nil, util.NewInternalServerError(err, "Failed to get resource reference: %v", err.Error())
}
if len(reference) == 0 {
return nil, util.NewResourcesNotFoundError(
@ -125,9 +128,9 @@ func (s *ResourceReferenceStore) GetResourceReference(resourceId string, resourc
func (s *ResourceReferenceStore) scanRows(r *sql.Rows) ([]model.ResourceReference, error) {
var references []model.ResourceReference
for r.Next() {
var resourceUUID, resourceType, referenceUUID, referenceType, relationship, payload string
var resourceUUID, resourceType, referenceUUID, referenceName, referenceType, relationship, payload string
err := r.Scan(
&resourceUUID, &resourceType, &referenceUUID, &referenceType, &relationship, &payload)
&resourceUUID, &resourceType, &referenceUUID, &referenceName, &referenceType, &relationship, &payload)
if err != nil {
return nil, err
}
@ -135,6 +138,7 @@ func (s *ResourceReferenceStore) scanRows(r *sql.Rows) ([]model.ResourceReferenc
ResourceUUID: resourceUUID,
ResourceType: common.ResourceType(resourceType),
ReferenceUUID: referenceUUID,
ReferenceName: referenceName,
ReferenceType: common.ResourceType(referenceType),
Relationship: common.Relationship(relationship),
Payload: payload,

18
backend/src/apiserver/storage/resource_reference_store_test.go
View File

@ -12,29 +12,29 @@ import (
var testRefOne = &model.ResourceReference{
ResourceUUID: "r1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
}
var testRefTwo = &model.ResourceReference{
ResourceUUID: "j2", ResourceType: common.Job,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName: "e2",
ReferenceType: common.Experiment, Relationship: common.Owner,
}
var testRefThree = &model.ResourceReference{
ResourceUUID: defaultFakeExpId, ResourceType: common.Experiment,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
Relationship: common.Owner,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName: "e2",
ReferenceType: common.Experiment, Relationship: common.Owner,
}
func TestResourceReferenceStore(t *testing.T) {
db := NewFakeDbOrFatal()
defer db.Close()
expStore := NewExperimentStore(db, util.NewFakeTimeForEpoch(), util.NewFakeUUIDGeneratorOrFatal(defaultFakeExpId, nil))
expStore.CreateExperiment(&model.Experiment{Name: "exp1"})
expStore.CreateExperiment(&model.Experiment{Name: "e1"})
expStore = NewExperimentStore(db, util.NewFakeTimeForEpoch(), util.NewFakeUUIDGeneratorOrFatal(defaultFakeExpIdTwo, nil))
expStore.CreateExperiment(&model.Experiment{Name: "exp2"})
expStore.CreateExperiment(&model.Experiment{Name: "e2"})
store := NewResourceReferenceStore(db)
// Create resource reference
@ -62,7 +62,7 @@ func TestResourceReferenceStore(t *testing.T) {
payload, err := json.Marshal(testRefOne)
assert.Equal(t, &model.ResourceReference{
ResourceUUID: "r1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1", ReferenceType: common.Experiment,
Relationship: common.Creator, Payload: string(payload)}, experimentRef)
// Delete resource references

84
backend/src/apiserver/storage/run_store_test.go
View File

@ -48,8 +48,8 @@ func initializeRunStore() (*DB, *RunStore) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -70,8 +70,8 @@ func initializeRunStore() (*DB, *RunStore) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -92,8 +92,8 @@ func initializeRunStore() (*DB, *RunStore) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "3", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpIdTwo, ReferenceName: "e2",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -124,8 +124,8 @@ func TestListRuns_Pagination(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
}}
@ -142,8 +142,8 @@ func TestListRuns_Pagination(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
}}
@ -222,8 +222,8 @@ func TestListRuns_Pagination_Descend(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
}}
@ -240,8 +240,8 @@ func TestListRuns_Pagination_Descend(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
}}
@ -283,8 +283,8 @@ func TestListRuns_Pagination_LessThanPageSize(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -300,8 +300,8 @@ func TestListRuns_Pagination_LessThanPageSize(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
}}
@ -344,8 +344,8 @@ func TestGetRun(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -392,8 +392,8 @@ func TestCreateOrUpdateRun_UpdateSuccess(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -429,8 +429,8 @@ func TestCreateOrUpdateRun_UpdateSuccess(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -466,6 +466,7 @@ func TestCreateOrUpdateRun_CreateSuccess(t *testing.T) {
ResourceUUID: "2000",
ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -492,6 +493,7 @@ func TestCreateOrUpdateRun_CreateSuccess(t *testing.T) {
ResourceUUID: "2000",
ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId,
ReferenceName: "e1",
ReferenceType: common.Experiment,
Relationship: common.Owner,
},
@ -560,8 +562,8 @@ func TestCreateOrUpdateRun_BadStorageStateValue(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -604,8 +606,8 @@ func TestTerminateRun(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
},
@ -683,13 +685,13 @@ func TestGetRun_InvalidMetricPayload_Ignore(t *testing.T) {
defer db.Close()
sql, args, _ := sq.
Insert("run_metrics").
SetMap(sq.Eq{
"RunUUID": "1",
"NodeID": "node1",
"Name": "accuracy",
"NumberValue": 0.88,
"Format": "RAW",
"Payload": "{ invalid; json,"}).ToSql()
SetMap(sq.Eq{
"RunUUID": "1",
"NodeID": "node1",
"Name": "accuracy",
"NumberValue": 0.88,
"Format": "RAW",
"Payload": "{ invalid; json,"}).ToSql()
db.Exec(sql, args...)
runDetail, err := runStore.GetRun("1")
@ -738,8 +740,8 @@ func TestListRuns_WithMetrics(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
Metrics: []*model.RunMetric{metric1, metric2},
@ -756,8 +758,8 @@ func TestListRuns_WithMetrics(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "2", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
Metrics: []*model.RunMetric{metric3},
@ -867,8 +869,8 @@ func TestArchiveRun_IncludedInRunList(t *testing.T) {
ResourceReferences: []*model.ResourceReference{
{
ResourceUUID: "1", ResourceType: common.Run,
ReferenceUUID: defaultFakeExpId, ReferenceType: common.Experiment,
Relationship: common.Creator,
ReferenceUUID: defaultFakeExpId, ReferenceName: "e1",
ReferenceType: common.Experiment, Relationship: common.Creator,
},
},
}}

30
backend/test/integration/job_api_test.go
View File

@ -97,12 +97,12 @@ func (s *JobApiTestSuite) TestJobApis() {
}}
helloWorldJob, err := s.jobClient.Create(createJobRequest)
assert.Nil(t, err)
s.checkHelloWorldJob(t, helloWorldJob, helloWorldExperiment.ID, helloWorldPipeline.ID)
s.checkHelloWorldJob(t, helloWorldJob, helloWorldExperiment.ID, helloWorldExperiment.Name, helloWorldPipeline.ID)
/* ---------- Get hello world job ---------- */
helloWorldJob, err = s.jobClient.Get(&jobparams.GetJobParams{ID: helloWorldJob.ID})
assert.Nil(t, err)
s.checkHelloWorldJob(t, helloWorldJob, helloWorldExperiment.ID, helloWorldPipeline.ID)
s.checkHelloWorldJob(t, helloWorldJob, helloWorldExperiment.ID, helloWorldExperiment.Name, helloWorldPipeline.ID)
/* ---------- Create a new argument parameter experiment ---------- */
experiment = &experiment_model.APIExperiment{Name: "argument parameter experiment"}
@ -133,7 +133,7 @@ func (s *JobApiTestSuite) TestJobApis() {
}}
argParamsJob, err := s.jobClient.Create(createJobRequest)
assert.Nil(t, err)
s.checkArgParamsJob(t, argParamsJob, argParamsExperiment.ID)
s.checkArgParamsJob(t, argParamsJob, argParamsExperiment.ID, argParamsExperiment.Name)
/* ---------- List all the jobs. Both jobs should be returned ---------- */
jobs, totalSize, _, err := s.jobClient.List(&jobparams.ListJobsParams{})
@ -196,7 +196,7 @@ func (s *JobApiTestSuite) TestJobApis() {
assert.Equal(t, 1, len(runs))
assert.Equal(t, 1, totalSize)
helloWorldRun := runs[0]
s.checkHelloWorldRun(t, helloWorldRun, helloWorldExperiment.ID, helloWorldJob.ID)
s.checkHelloWorldRun(t, helloWorldRun, helloWorldExperiment.ID, helloWorldExperiment.Name, helloWorldJob.ID, helloWorldJob.Name)
/* ---------- Check run for argument parameter job ---------- */
runs, totalSize, _, err = s.runClient.List(&runParams.ListRunsParams{
@ -206,7 +206,7 @@ func (s *JobApiTestSuite) TestJobApis() {
assert.Equal(t, 1, len(runs))
assert.Equal(t, 1, totalSize)
argParamsRun := runs[0]
s.checkArgParamsRun(t, argParamsRun, argParamsExperiment.ID, argParamsJob.ID)
s.checkArgParamsRun(t, argParamsRun, argParamsExperiment.ID, argParamsExperiment.Name, argParamsJob.ID, argParamsJob.Name)
/* ---------- Clean up ---------- */
test.DeleteAllExperiments(s.experimentClient, t)
@ -215,7 +215,7 @@ func (s *JobApiTestSuite) TestJobApis() {
test.DeleteAllRuns(s.runClient, t)
}
func (s *JobApiTestSuite) checkHelloWorldJob(t *testing.T, job *job_model.APIJob, experimentID string, pipelineID string) {
func (s *JobApiTestSuite) checkHelloWorldJob(t *testing.T, job *job_model.APIJob, experimentID string, experimentName string, pipelineID string) {
// Check workflow manifest is not empty
assert.Contains(t, job.PipelineSpec.WorkflowManifest, "whalesay")
expectedJob := &job_model.APIJob{
@ -228,7 +228,7 @@ func (s *JobApiTestSuite) checkHelloWorldJob(t *testing.T, job *job_model.APIJob
},
ResourceReferences: []*job_model.APIResourceReference{
{Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypeEXPERIMENT, ID: experimentID},
Relationship: job_model.APIRelationshipOWNER,
Name: experimentName, Relationship: job_model.APIRelationshipOWNER,
},
},
MaxConcurrency: 10,
@ -242,7 +242,7 @@ func (s *JobApiTestSuite) checkHelloWorldJob(t *testing.T, job *job_model.APIJob
assert.Equal(t, expectedJob, job)
}
func (s *JobApiTestSuite) checkArgParamsJob(t *testing.T, job *job_model.APIJob, experimentID string) {
func (s *JobApiTestSuite) checkArgParamsJob(t *testing.T, job *job_model.APIJob, experimentID string, experimentName string) {
argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
assert.Nil(t, err)
argParamsBytes, err = yaml.ToJSON(argParamsBytes)
@ -262,7 +262,7 @@ func (s *JobApiTestSuite) checkArgParamsJob(t *testing.T, job *job_model.APIJob,
},
ResourceReferences: []*job_model.APIResourceReference{
{Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypeEXPERIMENT, ID: experimentID},
Relationship: job_model.APIRelationshipOWNER,
Name: experimentName, Relationship: job_model.APIRelationshipOWNER,
},
},
MaxConcurrency: 10,
@ -276,31 +276,31 @@ func (s *JobApiTestSuite) checkArgParamsJob(t *testing.T, job *job_model.APIJob,
assert.Equal(t, expectedJob, job)
}
func (s *JobApiTestSuite) checkHelloWorldRun(t *testing.T, run *run_model.APIRun, experimentID string, jobID string) {
func (s *JobApiTestSuite) checkHelloWorldRun(t *testing.T, run *run_model.APIRun, experimentID string, experimentName string, jobID string, jobName string) {
// Check workflow manifest is not empty
assert.Contains(t, run.PipelineSpec.WorkflowManifest, "whalesay")
assert.Contains(t, run.Name, "helloworld")
// Check runtime workflow manifest is not empty
resourceReferences := []*run_model.APIResourceReference{
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: experimentID},
Relationship: run_model.APIRelationshipOWNER,
Name: experimentName, Relationship: run_model.APIRelationshipOWNER,
},
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeJOB, ID: jobID},
Relationship: run_model.APIRelationshipCREATOR,
Name: jobName, Relationship: run_model.APIRelationshipCREATOR,
},
}
assert.Equal(t, resourceReferences, run.ResourceReferences)
}
func (s *JobApiTestSuite) checkArgParamsRun(t *testing.T, run *run_model.APIRun, experimentID string, jobID string) {
func (s *JobApiTestSuite) checkArgParamsRun(t *testing.T, run *run_model.APIRun, experimentID string, experimentName string, jobID string, jobName string) {
assert.Contains(t, run.Name, "argumentparameter")
// Check runtime workflow manifest is not empty
resourceReferences := []*run_model.APIResourceReference{
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: experimentID},
Relationship: run_model.APIRelationshipOWNER,
Name: experimentName, Relationship: run_model.APIRelationshipOWNER,
},
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeJOB, ID: jobID},
Relationship: run_model.APIRelationshipCREATOR,
Name: jobName, Relationship: run_model.APIRelationshipCREATOR,
},
}
assert.Equal(t, resourceReferences, run.ResourceReferences)

22
backend/test/integration/run_api_test.go
View File

@ -81,17 +81,17 @@ func (s *RunApiTestSuite) TestRunApis() {
},
ResourceReferences: []*run_model.APIResourceReference{
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: helloWorldExperiment.ID},
Relationship: run_model.APIRelationshipOWNER},
Name: helloWorldExperiment.Name, Relationship: run_model.APIRelationshipOWNER},
},
}}
helloWorldRunDetail, _, err := s.runClient.Create(createRunRequest)
assert.Nil(t, err)
s.checkHelloWorldRunDetail(t, helloWorldRunDetail, helloWorldExperiment.ID, helloWorldPipeline.ID)
s.checkHelloWorldRunDetail(t, helloWorldRunDetail, helloWorldExperiment.ID, helloWorldExperiment.Name, helloWorldPipeline.ID)
/* ---------- Get hello world run ---------- */
helloWorldRunDetail, _, err = s.runClient.Get(&runparams.GetRunParams{RunID: helloWorldRunDetail.Run.ID})
assert.Nil(t, err)
s.checkHelloWorldRunDetail(t, helloWorldRunDetail, helloWorldExperiment.ID, helloWorldPipeline.ID)
s.checkHelloWorldRunDetail(t, helloWorldRunDetail, helloWorldExperiment.ID, helloWorldExperiment.Name, helloWorldPipeline.ID)
/* ---------- Create a new argument parameter experiment ---------- */
createExperimentRequest := &experimentparams.CreateExperimentParams{Body: &experiment_model.APIExperiment{Name: "argument parameter experiment"}}
@ -120,7 +120,7 @@ func (s *RunApiTestSuite) TestRunApis() {
}}
argParamsRunDetail, _, err := s.runClient.Create(createRunRequest)
assert.Nil(t, err)
s.checkArgParamsRunDetail(t, argParamsRunDetail, argParamsExperiment.ID)
s.checkArgParamsRunDetail(t, argParamsRunDetail, argParamsExperiment.ID, argParamsExperiment.Name)
/* ---------- List all the runs. Both runs should be returned ---------- */
runs, totalSize, _, err := s.runClient.List(&runparams.ListRunsParams{})
@ -214,7 +214,7 @@ func (s *RunApiTestSuite) TestRunApis() {
/* ---------- Get long-running run ---------- */
longRunningRunDetail, _, err = s.runClient.Get(&runparams.GetRunParams{RunID: longRunningRunDetail.Run.ID})
assert.Nil(t, err)
s.checkTerminatedRunDetail(t, longRunningRunDetail, helloWorldExperiment.ID, longRunningPipeline.ID)
s.checkTerminatedRunDetail(t, longRunningRunDetail, helloWorldExperiment.ID, helloWorldExperiment.Name, longRunningPipeline.ID)
/* ---------- Clean up ---------- */
test.DeleteAllExperiments(s.experimentClient, t)
@ -222,7 +222,7 @@ func (s *RunApiTestSuite) TestRunApis() {
test.DeleteAllRuns(s.runClient, t)
}
func (s *RunApiTestSuite) checkTerminatedRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string, pipelineId string) {
func (s *RunApiTestSuite) checkTerminatedRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string, experimentName string, pipelineId string) {
// Check workflow manifest is not empty
assert.Contains(t, runDetail.Run.PipelineSpec.WorkflowManifest, "wait-awhile")
// Check runtime workflow manifest is not empty
@ -239,7 +239,7 @@ func (s *RunApiTestSuite) checkTerminatedRunDetail(t *testing.T, runDetail *run_
},
ResourceReferences: []*run_model.APIResourceReference{
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: experimentId},
Relationship: run_model.APIRelationshipOWNER,
Name: experimentName, Relationship: run_model.APIRelationshipOWNER,
},
},
CreatedAt: runDetail.Run.CreatedAt,
@ -249,7 +249,7 @@ func (s *RunApiTestSuite) checkTerminatedRunDetail(t *testing.T, runDetail *run_
assert.Equal(t, expectedRun, runDetail.Run)
}
func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string, pipelineId string) {
func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string, experimentName string, pipelineId string) {
// Check workflow manifest is not empty
assert.Contains(t, runDetail.Run.PipelineSpec.WorkflowManifest, "whalesay")
// Check runtime workflow manifest is not empty
@ -266,7 +266,7 @@ func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, runDetail *run_
},
ResourceReferences: []*run_model.APIResourceReference{
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: experimentId},
Relationship: run_model.APIRelationshipOWNER,
Name: experimentName, Relationship: run_model.APIRelationshipOWNER,
},
},
CreatedAt: runDetail.Run.CreatedAt,
@ -276,7 +276,7 @@ func (s *RunApiTestSuite) checkHelloWorldRunDetail(t *testing.T, runDetail *run_
assert.Equal(t, expectedRun, runDetail.Run)
}
func (s *RunApiTestSuite) checkArgParamsRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string) {
func (s *RunApiTestSuite) checkArgParamsRunDetail(t *testing.T, runDetail *run_model.APIRunDetail, experimentId string, experimentName string) {
argParamsBytes, err := ioutil.ReadFile("../resources/arguments-parameters.yaml")
assert.Nil(t, err)
argParamsBytes, err = yaml.ToJSON(argParamsBytes)
@ -297,7 +297,7 @@ func (s *RunApiTestSuite) checkArgParamsRunDetail(t *testing.T, runDetail *run_m
},
ResourceReferences: []*run_model.APIResourceReference{
{Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: experimentId},
Relationship: run_model.APIRelationshipOWNER,
Name: experimentName, Relationship: run_model.APIRelationshipOWNER,
},
},
CreatedAt: runDetail.Run.CreatedAt,

2
sdk/python/setup.py
View File

@ -30,7 +30,7 @@ REQUIRES = [
'google-auth>=1.6.1',
'requests_toolbelt>=0.8.0',
'cloudpickle',
'kfp-server-api >= 0.1.18, < 0.1.19', #Update the upper version whenever a new version of the kfp-server-api package is released. Update the lower version when there is a breaking change in kfp-server-api.
'kfp-server-api >= 0.1.18, <= 0.1.25', #Update the upper version whenever a new version of the kfp-server-api package is released. Update the lower version when there is a breaking change in kfp-server-api.
'argo-models == 2.2.1a', #2.2.1a is equivalent to argo 2.2.1
'jsonschema >= 3.0.1',
'tabulate == 0.8.3',