From 1fb834edc20917a924c90cd917abdca6b02ab2a7 Mon Sep 17 00:00:00 2001 From: yangjun289519474 <31427878+yangjun289519474@users.noreply.github.com> Date: Fri, 4 Jun 2021 11:03:44 +0800 Subject: [PATCH] refactor(manager): modify mysql table schema, orm json tag. (#283) Signed-off-by: hanson.yj Co-authored-by: hanson.yj --- go.mod | 1 + go.sum | 2 + manager/apis/v2/handler/scheduler_cluster.go | 4 +- manager/apis/v2/types/schema.go | 16 +++---- manager/server/server_test.go | 22 ++++----- manager/server/service/manager_server.go | 10 ++-- manager/store/orm/cdn_cluster.go | 20 ++++---- manager/store/orm/cdn_instance.go | 31 ++++++------ manager/store/orm/orm_store.go | 3 +- manager/store/orm/scheduler_cluster.go | 50 ++++++++++---------- manager/store/orm/scheduler_instance.go | 7 +-- manager/store/orm/security_domain.go | 4 +- 12 files changed, 91 insertions(+), 79 deletions(-) diff --git a/go.mod b/go.mod index 630212615..cc421394d 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/golang/protobuf v1.4.3 github.com/google/uuid v1.1.5 github.com/gorilla/mux v1.7.3 + github.com/iancoleman/strcase v0.1.3 github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.9 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db diff --git a/go.sum b/go.sum index 3d4de1e56..ad7802c4f 100644 --- a/go.sum +++ b/go.sum @@ -231,6 +231,8 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.1.3 h1:dJBk1m2/qjL1twPLf68JND55vvivMupZ4wIzE8CTdBw= +github.com/iancoleman/strcase v0.1.3/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= diff --git a/manager/apis/v2/handler/scheduler_cluster.go b/manager/apis/v2/handler/scheduler_cluster.go index 03bb7ab25..0d299307d 100644 --- a/manager/apis/v2/handler/scheduler_cluster.go +++ b/manager/apis/v2/handler/scheduler_cluster.go @@ -200,8 +200,8 @@ func (handler *Handler) ListSchedulerClusters(ctx *gin.Context) { } func checkSchedulerClusterValidate(cluster *types.SchedulerCluster) (err error) { - var schedulerConfigMap map[string]string - err = json.Unmarshal([]byte(cluster.SchedulerConfig), &schedulerConfigMap) + var configMap map[string]string + err = json.Unmarshal([]byte(cluster.Config), &configMap) if err != nil { err = errors.New("unmarshal scheduler_config error: scheduler_config must map[string]string") return diff --git a/manager/apis/v2/types/schema.go b/manager/apis/v2/types/schema.go index 7e28eff68..1463d21fd 100644 --- a/manager/apis/v2/types/schema.go +++ b/manager/apis/v2/types/schema.go @@ -1,14 +1,14 @@ package types type SchedulerCluster struct { - ClusterID string `json:"cluster_id" binding:"omitempty"` - SchedulerConfig string `json:"scheduler_config" binding:"required"` - ClientConfig string `json:"client_config" binding:"required"` - Version int64 `json:"version" binding:"omitempty"` - Creator string `json:"creator" binding:"omitempty"` - Modifier string `json:"modifier" binding:"omitempty"` - CreatedAt string `json:"created_at" binding:"omitempty"` - UpdatedAt string `json:"updated_at" binding:"omitempty"` + ClusterID string `json:"cluster_id" binding:"omitempty"` + Config string `json:"config" binding:"required"` + ClientConfig string `json:"client_config" binding:"required"` + Version int64 `json:"version" binding:"omitempty"` + Creator string `json:"creator" binding:"omitempty"` + Modifier string `json:"modifier" binding:"omitempty"` + CreatedAt string `json:"created_at" binding:"omitempty"` + UpdatedAt string `json:"updated_at" binding:"omitempty"` } type SchedulerClusterURI struct { diff --git a/manager/server/server_test.go b/manager/server/server_test.go index 068750458..02f6507bd 100644 --- a/manager/server/server_test.go +++ b/manager/server/server_test.go @@ -42,8 +42,8 @@ func (suite *ServerTestSuite) testDefaultSchedulerCluster() *types.SchedulerClus clientConfigBytes, _ := json.Marshal(&clientConfigMap) return &types.SchedulerCluster{ - SchedulerConfig: string(schedulerConfigBytes), - ClientConfig: string(clientConfigBytes), + Config: string(schedulerConfigBytes), + ClientConfig: string(clientConfigBytes), } } @@ -269,7 +269,7 @@ func (suite *ServerTestSuite) TestGetSchedulerClusterConfig() { assert.Nil(err) assert.NotNil(ret) cfg := ret.GetSchedulerConfig() - assert.Equal(cluster.SchedulerConfig, cfg.ClusterConfig) + assert.Equal(cluster.Config, cfg.ClusterConfig) assert.Equal(cluster.ClientConfig, cfg.ClientConfig) } @@ -313,13 +313,13 @@ func (suite *ServerTestSuite) TestGetSchedulerClusterConfig() { { var schedulerConfigMap map[string]string - err := json.Unmarshal([]byte(cluster.SchedulerConfig), &schedulerConfigMap) + err := json.Unmarshal([]byte(cluster.Config), &schedulerConfigMap) assert.Nil(err) schedulerConfigMap["CDN_CLUSTER_ID"] = cdnCluster.ClusterID schedulerConfigByte, err := json.Marshal(schedulerConfigMap) assert.Nil(err) - cluster.SchedulerConfig = string(schedulerConfigByte) + cluster.Config = string(schedulerConfigByte) suite.server.ms.UpdateSchedulerCluster(context.TODO(), cluster) } @@ -334,7 +334,7 @@ func (suite *ServerTestSuite) TestGetSchedulerClusterConfig() { assert.Nil(err) assert.NotNil(ret) cfg := ret.GetSchedulerConfig() - assert.Equal(cluster.SchedulerConfig, cfg.ClusterConfig) + assert.Equal(cluster.Config, cfg.ClusterConfig) assert.Equal(cluster.ClientConfig, cfg.ClientConfig) cdnHost := cfg.GetCdnHosts() @@ -407,25 +407,25 @@ func (suite *ServerTestSuite) TestSchedulerCluster() { assert.NotNil(ret) assert.Nil(err) assert.Equal(cluster.ClusterID, ret.ClusterID) - assert.Equal(cluster.SchedulerConfig, ret.SchedulerConfig) + assert.Equal(cluster.Config, ret.Config) assert.Equal(cluster.ClientConfig, ret.ClientConfig) } { var schedulerConfigMap map[string]string - err := json.Unmarshal([]byte(cluster.SchedulerConfig), &schedulerConfigMap) + err := json.Unmarshal([]byte(cluster.Config), &schedulerConfigMap) assert.Nil(err) schedulerConfigMap["schedulerConfig_a"] = "schedulerConfig_a_update" schedulerConfigByte, err := json.Marshal(schedulerConfigMap) assert.Nil(err) - cluster.SchedulerConfig = string(schedulerConfigByte) + cluster.Config = string(schedulerConfigByte) ret, err := suite.server.ms.UpdateSchedulerCluster(context.TODO(), cluster) assert.NotNil(ret) assert.Nil(err) assert.Equal(cluster.ClusterID, ret.ClusterID) - assert.Equal(cluster.SchedulerConfig, ret.SchedulerConfig) + assert.Equal(cluster.Config, ret.Config) assert.Equal(cluster.ClientConfig, ret.ClientConfig) } @@ -436,7 +436,7 @@ func (suite *ServerTestSuite) TestSchedulerCluster() { for i, c := range ret { if cluster.ClusterID == c.ClusterID { assert.Equal(cluster.ClusterID, ret[i].ClusterID) - assert.Equal(cluster.SchedulerConfig, ret[i].SchedulerConfig) + assert.Equal(cluster.Config, ret[i].Config) assert.Equal(cluster.ClientConfig, ret[i].ClientConfig) } } diff --git a/manager/server/service/manager_server.go b/manager/server/service/manager_server.go index ede4b648b..7b1e5a926 100644 --- a/manager/server/service/manager_server.go +++ b/manager/server/service/manager_server.go @@ -142,17 +142,17 @@ func (ms *ManagerServer) GetClusterConfig(ctx context.Context, req *manager.GetC } else if manager.ResourceType_Scheduler == req.GetType() { schedulerInstance := interInstance.(*types.SchedulerInstance) schedulerCluster := interCluster.(*types.SchedulerCluster) - schedulerConfigsMap := make(map[string]interface{}) - err := json.Unmarshal([]byte(schedulerCluster.SchedulerConfig), &schedulerConfigsMap) + configsMap := make(map[string]interface{}) + err := json.Unmarshal([]byte(schedulerCluster.Config), &configsMap) if err != nil { return nil, err } - CDNClusterID, exist := schedulerConfigsMap["CDN_CLUSTER_ID"] + CDNClusterID, exist := configsMap["CDN_CLUSTER_ID"] if !exist { return &manager.ClusterConfig{Config: &manager.ClusterConfig_SchedulerConfig{SchedulerConfig: &manager.SchedulerConfig{ ClusterId: schedulerCluster.ClusterID, - ClusterConfig: schedulerCluster.SchedulerConfig, + ClusterConfig: schedulerCluster.Config, ClientConfig: schedulerCluster.ClientConfig, ClusterVersion: schedulerCluster.Version, InstanceConfig: schedulerInstance.IDC, // todo InstanceConfig format @@ -218,7 +218,7 @@ func (ms *ManagerServer) GetClusterConfig(ctx context.Context, req *manager.GetC return &manager.ClusterConfig{Config: &manager.ClusterConfig_SchedulerConfig{SchedulerConfig: &manager.SchedulerConfig{ ClusterId: schedulerCluster.ClusterID, - ClusterConfig: schedulerCluster.SchedulerConfig, + ClusterConfig: schedulerCluster.Config, ClientConfig: schedulerCluster.ClientConfig, ClusterVersion: schedulerCluster.Version, InstanceConfig: schedulerInstance.IDC, // todo InstanceConfig format diff --git a/manager/store/orm/cdn_cluster.go b/manager/store/orm/cdn_cluster.go index cb18a9e09..25da81053 100644 --- a/manager/store/orm/cdn_cluster.go +++ b/manager/store/orm/cdn_cluster.go @@ -13,15 +13,17 @@ import ( ) type CDNClusterTable struct { - ID uint `gorm:"column:id;primaryKey"` - ClusterID string `gorm:"column:cluster_id;unique;size:63"` - Config string `gorm:"column:config;size:4095"` - Creator string `gorm:"column:creator;size:31"` - Modifier string `gorm:"column:modifier;size:31"` - Version int64 `gorm:"column:version"` - CreatedAt time.Time `gorm:"column:created_at"` - UpdatedAt time.Time `gorm:"column:updated_at"` - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` + ID uint `gorm:"column:id;primaryKey"` + ClusterID string `gorm:"column:cluster_id;unique;size:63"` + Config string `gorm:"column:config;type:text;not null"` + Creator string `gorm:"column:creator;size:31"` + Modifier string `gorm:"column:modifier;size:31"` + Enable bool `gorm:"column:enable;type:bool;"` + Version int64 `gorm:"column:version"` + Description string `gorm:"column:description;size:1023"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` } type CDNClusterStore struct { diff --git a/manager/store/orm/cdn_instance.go b/manager/store/orm/cdn_instance.go index ab0556098..06c74bade 100644 --- a/manager/store/orm/cdn_instance.go +++ b/manager/store/orm/cdn_instance.go @@ -13,21 +13,22 @@ import ( ) type CDNInstanceTable struct { - ID uint `gorm:"column:id;primaryKey"` - InstanceID string `gorm:"column:instance_id;unique;size:63"` - ClusterID string `gorm:"column:cluster_id;size:63"` - IDC string `gorm:"column:idc;size:63"` - Location string `gorm:"column:location;size:4095"` - HostName string `gorm:"column:host_name;size:63"` - IP string `gorm:"column:ip;size:31"` - Port int32 `gorm:"column:port"` - RPCPort int32 `gorm:"column:rpc_port"` - DownPort int32 `gorm:"column:down_port"` - State string `gorm:"column:state;size:15"` - Version int64 `gorm:"column:version"` - CreatedAt time.Time `gorm:"column:created_at"` - UpdatedAt time.Time `gorm:"column:updated_at"` - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` + ID uint `gorm:"column:id;primaryKey"` + InstanceID string `gorm:"column:instance_id;unique;size:63"` + ClusterID string `gorm:"column:cluster_id;size:63"` + IDC string `gorm:"column:idc;size:63"` + Location string `gorm:"column:location;type:text;not null"` + HostName string `gorm:"column:host_name;size:63"` + IP string `gorm:"column:ip;size:31"` + Port int32 `gorm:"column:port"` + RPCPort int32 `gorm:"column:rpc_port"` + DownPort int32 `gorm:"column:down_port"` + State string `gorm:"column:state;size:15"` + Version int64 `gorm:"column:version"` + Description string `gorm:"column:description;size:1023"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` } type CDNInstanceStore struct { diff --git a/manager/store/orm/orm_store.go b/manager/store/orm/orm_store.go index 67f4dab21..de1129305 100644 --- a/manager/store/orm/orm_store.go +++ b/manager/store/orm/orm_store.go @@ -8,6 +8,7 @@ import ( "d7y.io/dragonfly/v2/manager/store" "d7y.io/dragonfly/v2/pkg/dfcodes" "d7y.io/dragonfly/v2/pkg/dferrors" + "github.com/iancoleman/strcase" "github.com/xo/dburl" "gorm.io/driver/mysql" "gorm.io/gorm" @@ -65,7 +66,7 @@ func newOrmStore(cfg *config.StoreConfig) (*ormStore, error) { } for t, f := range ormTables { - table := t.String() + table := strcase.ToSnake(t.String()) s, err := f(db, table) if err != nil { return nil, err diff --git a/manager/store/orm/scheduler_cluster.go b/manager/store/orm/scheduler_cluster.go index 213882d5e..2d9ff4ea0 100644 --- a/manager/store/orm/scheduler_cluster.go +++ b/manager/store/orm/scheduler_cluster.go @@ -13,16 +13,18 @@ import ( ) type SchedulerClusterTable struct { - ID uint `gorm:"column:id;primaryKey"` - ClusterID string `gorm:"column:cluster_id;unique;size:63"` - SchedulerConfig string `gorm:"column:scheduler_config;size:4095"` - ClientConfig string `gorm:"column:client_config;size:4095"` - Creator string `gorm:"column:creator;size:31"` - Modifier string `gorm:"column:modifier;size:31"` - Version int64 `gorm:"column:version"` - CreatedAt time.Time `gorm:"column:created_at"` - UpdatedAt time.Time `gorm:"column:updated_at"` - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` + ID uint `gorm:"column:id;primaryKey"` + ClusterID string `gorm:"column:cluster_id;unique;size:63"` + Config string `gorm:"column:config;type:text;not null"` + ClientConfig string `gorm:"column:client_config;type:text;not null"` + Creator string `gorm:"column:creator;size:31"` + Modifier string `gorm:"column:modifier;size:31"` + Enable bool `gorm:"column:enable;type:bool"` + Version int64 `gorm:"column:version"` + Description string `gorm:"column:description;size:1023"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` } type SchedulerClusterStore struct { @@ -48,30 +50,30 @@ func NewSchedulerClusterStore(db *gorm.DB, table string) (store.Store, error) { func SchedulerClusterToTable(t *types.SchedulerCluster) *SchedulerClusterTable { return &SchedulerClusterTable{ - ClusterID: t.ClusterID, - SchedulerConfig: t.SchedulerConfig, - ClientConfig: t.ClientConfig, - Creator: t.Creator, - Modifier: t.Modifier, - Version: time.Now().UnixNano(), + ClusterID: t.ClusterID, + Config: t.Config, + ClientConfig: t.ClientConfig, + Creator: t.Creator, + Modifier: t.Modifier, + Version: time.Now().UnixNano(), } } func SchedulerClusterToSchema(t *SchedulerClusterTable) *types.SchedulerCluster { return &types.SchedulerCluster{ - ClusterID: t.ClusterID, - SchedulerConfig: t.SchedulerConfig, - ClientConfig: t.ClientConfig, - Creator: t.Creator, - Modifier: t.Modifier, - CreatedAt: t.CreatedAt.String(), - UpdatedAt: t.UpdatedAt.String(), + ClusterID: t.ClusterID, + Config: t.Config, + ClientConfig: t.ClientConfig, + Creator: t.Creator, + Modifier: t.Modifier, + CreatedAt: t.CreatedAt.String(), + UpdatedAt: t.UpdatedAt.String(), } } func (s *SchedulerClusterStore) updateSchemaToTable(new, old *SchedulerClusterTable) *SchedulerClusterTable { new.ID = old.ID - if new.SchedulerConfig != old.SchedulerConfig || new.ClientConfig != old.ClientConfig { + if new.Config != old.Config || new.ClientConfig != old.ClientConfig { new.Version = time.Now().UnixNano() } else { new.Version = old.Version diff --git a/manager/store/orm/scheduler_instance.go b/manager/store/orm/scheduler_instance.go index 2d8360c8b..be36cfac5 100644 --- a/manager/store/orm/scheduler_instance.go +++ b/manager/store/orm/scheduler_instance.go @@ -17,15 +17,16 @@ type SchedulerInstanceTable struct { InstanceID string `gorm:"column:instance_id;unique;size:63"` ClusterID string `gorm:"column:cluster_id;size:63"` SecurityDomain string `gorm:"column:security_domain;size:63"` - VIPs string `gorm:"column:vips;size:4095"` + VIPs string `gorm:"column:vips;type:text;not null"` IDC string `gorm:"column:idc;size:63"` - Location string `gorm:"column:location;size:4095"` - NetConfig string `gorm:"column:net_config;size:4095"` + Location string `gorm:"column:location;type:text;not null"` + NetConfig string `gorm:"column:net_config;type:text;not null"` HostName string `gorm:"column:host_name;size:63"` IP string `gorm:"column:ip;size:31"` Port int32 `gorm:"column:port"` State string `gorm:"column:state;size:15"` Version int64 `gorm:"column:version"` + Description string `gorm:"column:description;size:1023"` CreatedAt time.Time `gorm:"column:created_at"` UpdatedAt time.Time `gorm:"column:updated_at"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"` diff --git a/manager/store/orm/security_domain.go b/manager/store/orm/security_domain.go index 3009aa028..ceb0a6b37 100644 --- a/manager/store/orm/security_domain.go +++ b/manager/store/orm/security_domain.go @@ -16,10 +16,12 @@ type SecurityDomainTable struct { ID uint `gorm:"column:id;primaryKey"` SecurityDomain string `gorm:"column:security_domain;unique;size:63"` DisplayName string `gorm:"column:display_name;size:63"` - ProxyDomain string `gorm:"column:proxy_domain;size:4095"` + ProxyDomain string `gorm:"column:proxy_domain;type:text;not null"` + Config string `gorm:"column:config;type:text;not null"` Creator string `gorm:"column:creator;size:31"` Modifier string `gorm:"column:modifier;size:31"` Version int64 `gorm:"column:version"` + Description string `gorm:"column:description;size:1023"` CreatedAt time.Time `gorm:"column:created_at"` UpdatedAt time.Time `gorm:"column:updated_at"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index"`