diff --git a/Makefile b/Makefile index a1da75ae9..03f5e4d15 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PROJECT_NAME := "d7y.io/dragonfly/v2" DFGET_NAME := "dfget" VERSION := "2.0.0" PKG := "$(PROJECT_NAME)" -PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/manager/\)' | grep -v '\(/test/\)') +PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/test/\)') GIT_COMMIT := $(shell git rev-parse --verify HEAD --short=7) GIT_COMMIT_LONG := $(shell git rev-parse --verify HEAD) DFGET_ARCHIVE_PREFIX := "$(DFGET_NAME)_$(GIT_COMMIT)" diff --git a/go.mod b/go.mod index 036693bee..034689cd3 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 golang.org/x/tools v0.1.4 // indirect + gonum.org/v1/gonum v0.9.3 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect google.golang.org/grpc v1.36.0 google.golang.org/protobuf v1.26.0 diff --git a/go.sum b/go.sum index deeb7f405..caecb4965 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -25,6 +26,7 @@ github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw= github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -47,6 +49,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -94,6 +97,8 @@ github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XW github.com/envoyproxy/protoc-gen-validate v0.6.1/go.mod h1:txg5va2Qkip90uYoSKH+nkAAmXrb2j3iq4FLwdrCbXQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -107,12 +112,17 @@ github.com/go-echarts/go-echarts/v2 v2.2.3 h1:H8oPdUpzuiV2K8S4xYZa1JRNjP3U0h7HVq github.com/go-echarts/go-echarts/v2 v2.2.3/go.mod h1:6TOomEztzGDVDkOSCFBq3ed7xOYfbOqhaBzD0YV771A= github.com/go-echarts/statsview v0.3.4 h1:CCuytRAutdnF901NrR4BzSjHXjUp8OyA3/iopgG/1/Y= github.com/go-echarts/statsview v0.3.4/go.mod h1:AehKjL9cTFMeIo5QdV8sQO43vFmfY65X5GMWa3XMciY= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -151,6 +161,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -322,6 +333,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -431,6 +444,8 @@ github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNC github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -473,6 +488,7 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -598,16 +614,27 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20201221025956-e89b829e73ea h1:GnGfrp0fiNhiBS/v/aCFTmfEWgkvxW4Qiu8oM2/IfZ4= golang.org/x/exp v0.0.0-20201221025956-e89b829e73ea/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -706,6 +733,7 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -720,6 +748,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -727,8 +756,10 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -745,6 +776,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -768,6 +800,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3 h1:DnoIG+QAMaF5NvxnGe/oKsgKcAc6PcUyl8q0VetfQ8s= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -872,6 +912,7 @@ k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/internal/dfplugin/dfplugin.go b/internal/dfplugin/dfplugin.go index 5f11212c1..a14eac9aa 100644 --- a/internal/dfplugin/dfplugin.go +++ b/internal/dfplugin/dfplugin.go @@ -34,6 +34,7 @@ const ( // PluginMetaKeyType indicates the type of a plugin, currently support: resource PluginMetaKeyType = "type" + // PluginMetaKeyName indicates the name of a plugin PluginMetaKeyName = "name" ) @@ -42,6 +43,7 @@ type PluginType string const ( PluginTypeResource = PluginType("resource") + PluginTypeManager = PluginType("manager") ) type PluginInitFunc func(option map[string]string) (plugin interface{}, meta map[string]string, err error) diff --git a/manager/model/scheduler_cluster.go b/manager/model/scheduler_cluster.go index 0b1703f92..2c7b36831 100644 --- a/manager/model/scheduler_cluster.go +++ b/manager/model/scheduler_cluster.go @@ -10,6 +10,7 @@ type SchedulerCluster struct { BIO string `gorm:"column:bio;size:1024" json:"bio"` Config datatypes.JSONMap `gorm:"column:config;not null" json:"config"` ClientConfig datatypes.JSONMap `gorm:"column:client_config;not null" json:"client_config"` + Scopes datatypes.JSONMap `gorm:"column:scopes" json:"scopes"` IsDefault bool `gorm:"column:is_default;not null;default:false" json:"is_default"` CDNClusters []CDNCluster `gorm:"many2many:cdn_cluster_scheduler_cluster;" json:"-"` Schedulers []Scheduler `json:"-"` diff --git a/manager/searcher/plugin.go b/manager/searcher/plugin.go new file mode 100644 index 000000000..fb0c74206 --- /dev/null +++ b/manager/searcher/plugin.go @@ -0,0 +1,39 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package searcher + +import ( + "errors" + + "d7y.io/dragonfly/v2/internal/dfplugin" +) + +const ( + pluginName = "searcher" +) + +func LoadPlugin() (Searcher, error) { + client, _, err := dfplugin.Load(dfplugin.PluginTypeManager, pluginName, map[string]string{}) + if err != nil { + return nil, err + } + + if rc, ok := client.(Searcher); ok { + return rc, err + } + return nil, errors.New("invalid client, not a ResourceClient") +} diff --git a/manager/searcher/plugin_test.go b/manager/searcher/plugin_test.go new file mode 100644 index 000000000..7f3d7618d --- /dev/null +++ b/manager/searcher/plugin_test.go @@ -0,0 +1,72 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package searcher + +import ( + "os" + "os/exec" + "path" + "testing" + + testifyassert "github.com/stretchr/testify/assert" +) + +func TestLoadPlugin(t *testing.T) { + assert := testifyassert.New(t) + defer func() { + os.Remove("./testdata/d7y-manager-plugin-searcher.so") + os.Remove("./testdata/test") + }() + + var ( + cmd *exec.Cmd + output []byte + wd string + err error + ) + + // build plugin + cmd = exec.Command("go", "build", "-buildmode=plugin", "-o=./testdata/d7y-manager-plugin-searcher.so", "testdata/plugin/searcher.go") + output, err = cmd.CombinedOutput() + assert.Nil(err) + if err != nil { + t.Fatalf(string(output)) + return + } + + // build test binary + cmd = exec.Command("go", "build", "-o=./testdata/test", "testdata/main.go") + output, err = cmd.CombinedOutput() + assert.Nil(err) + if err != nil { + t.Fatalf(string(output)) + return + } + + wd, err = os.Getwd() + assert.Nil(err) + wd = path.Join(wd, "testdata") + + // execute test binary + cmd = exec.Command("./testdata/test", "-plugin-dir", wd) + output, err = cmd.CombinedOutput() + assert.Nil(err) + if err != nil { + t.Fatalf(string(output)) + return + } +} diff --git a/manager/searcher/search.go b/manager/searcher/search.go new file mode 100644 index 000000000..a750d0a39 --- /dev/null +++ b/manager/searcher/search.go @@ -0,0 +1,123 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package searcher + +import ( + "sort" + + "d7y.io/dragonfly/v2/manager/model" + "github.com/mitchellh/mapstructure" + "gonum.org/v1/gonum/stat" +) + +const ( + conditionSecurityDomain = "security_domain" + conditionLocation = "location" + conditionIDC = "idc" +) + +const ( + conditionLocationWeight = 0.7 + conditionIDCWeight = 0.3 +) + +type Scopes struct { + Location []string `mapstructure:"location"` + IDC []string `mapstructure:"idc"` +} + +type Searcher interface { + FindSchedulerCluster([]model.SchedulerCluster, map[string]string) (model.SchedulerCluster, bool) +} + +type searcher struct{} + +func New() Searcher { + s, err := LoadPlugin() + if err != nil { + return &searcher{} + } + + return s +} + +func (s *searcher) FindSchedulerCluster(schedulerClusters []model.SchedulerCluster, conditions map[string]string) (model.SchedulerCluster, bool) { + if len(schedulerClusters) <= 0 || len(conditions) <= 0 { + return model.SchedulerCluster{}, false + } + + // If there are security domain conditions, match clusters of the same security domain. + // If the security domain condition does not exist, it matches clusters that does not have a security domain. + // Then use clusters sets to score according to scopes. + securityDomain := conditions[conditionSecurityDomain] + var clusters []model.SchedulerCluster + for _, v := range schedulerClusters { + if v.SecurityGroup.Domain == securityDomain { + clusters = append(clusters, v) + } + } + + switch len(clusters) { + case 0: + return model.SchedulerCluster{}, false + case 1: + return clusters[0], true + default: + var maxMean float64 = 0 + cluster := clusters[0] + for _, v := range clusters { + mean := calculateSchedulerClusterMean(conditions, v.Scopes) + if mean > maxMean { + maxMean = mean + cluster = v + } + } + return cluster, true + } +} + +func calculateSchedulerClusterMean(conditions map[string]string, rawScopes map[string]interface{}) float64 { + var scopes Scopes + if err := mapstructure.Decode(rawScopes, &scopes); err != nil { + return 0 + } + + location := conditions[conditionLocation] + lx := calculateConditionScore(location, scopes.Location) + + idc := conditions[conditionIDC] + ix := calculateConditionScore(idc, scopes.IDC) + + return stat.Mean([]float64{lx, ix}, []float64{conditionLocationWeight, conditionIDCWeight}) +} + +func calculateConditionScore(value string, scope []string) float64 { + if value == "" { + return 0 + } + + if len(scope) <= 0 { + return 0 + } + + i := sort.SearchStrings(scope, value) + if i < 0 { + return 0 + } + + return 1 +} diff --git a/manager/searcher/search_test.go b/manager/searcher/search_test.go new file mode 100644 index 000000000..94201073a --- /dev/null +++ b/manager/searcher/search_test.go @@ -0,0 +1,348 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package searcher + +import ( + "testing" + + "d7y.io/dragonfly/v2/manager/model" + "github.com/stretchr/testify/assert" +) + +func TestSchedulerCluster(t *testing.T) { + tests := []struct { + name string + schedulerClusters []model.SchedulerCluster + conditions map[string]string + expect func(t *testing.T, data model.SchedulerCluster, ok bool) + }{ + { + name: "conditions is empty", + schedulerClusters: []model.SchedulerCluster{{Name: "foo"}}, + conditions: map[string]string{}, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(ok, false) + }, + }, + { + name: "scheduler clusters is empty", + schedulerClusters: []model.SchedulerCluster{}, + conditions: map[string]string{"location": "foo"}, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(ok, false) + }, + }, + { + name: "match according to security_domain condition", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + SecurityGroup: model.SecurityGroup{ + Domain: "domain-1", + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{"security_domain": "domain-1"}, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + { + name: "match according to location condition", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + Scopes: map[string]interface{}{ + "location": []string{"location-1"}, + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{"location": "location-1"}, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + { + name: "match according to idc condition", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + Scopes: map[string]interface{}{ + "idc": []string{"idc-1"}, + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{"idc": "idc-1"}, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + { + name: "match according to location and idc condition", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + Scopes: map[string]interface{}{ + "location": []string{"location-1"}, + "idc": []string{"idc-1"}, + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{ + "location": "location-1", + "idc": "idc-1", + }, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + { + name: "match according to security_domain and location conditions", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + Scopes: map[string]interface{}{ + "location": []string{"location-1"}, + }, + SecurityGroup: model.SecurityGroup{ + Domain: "domain-1", + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{ + "security_domain": "domain-1", + "location": "location-1", + }, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + { + name: "match according to security_domain and idc conditions", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + Scopes: map[string]interface{}{ + "idc": []string{"idc-1"}, + }, + SecurityGroup: model.SecurityGroup{ + Domain: "domain-1", + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{ + "security_domain": "domain-1", + "idc": "idc-1", + }, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + { + name: "match according to all conditions", + schedulerClusters: []model.SchedulerCluster{ + { + Name: "foo", + Scopes: map[string]interface{}{ + "idc": []string{"idc-1"}, + "location": []string{"location-1"}, + }, + SecurityGroup: model.SecurityGroup{ + Domain: "domain-1", + }, + }, + { + Name: "bar", + }, + }, + conditions: map[string]string{ + "security_domain": "domain-1", + "idc": "idc-1", + "location": "location-1", + }, + expect: func(t *testing.T, data model.SchedulerCluster, ok bool) { + assert := assert.New(t) + assert.Equal(data.Name, "foo") + assert.Equal(ok, true) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + searcher := New() + clusters, ok := searcher.FindSchedulerCluster(tc.schedulerClusters, tc.conditions) + tc.expect(t, clusters, ok) + }) + } +} + +func TestCalculateSchedulerClusterMean(t *testing.T) { + tests := []struct { + name string + conditions map[string]string + rawScopes map[string]interface{} + expect func(t *testing.T, mean float64) + }{ + { + name: "conditions and rawScopes is empty", + conditions: map[string]string{}, + rawScopes: map[string]interface{}{}, + expect: func(t *testing.T, mean float64) { + assert := assert.New(t) + assert.Equal(mean, float64(0)) + }, + }, + { + name: "missed matches", + conditions: map[string]string{ + "location": "location-1", + }, + rawScopes: map[string]interface{}{ + "idc": []string{"idc-1"}, + }, + expect: func(t *testing.T, mean float64) { + assert := assert.New(t) + assert.Equal(mean, float64(0)) + }, + }, + { + name: "match according to location", + conditions: map[string]string{ + "location": "location-1", + }, + rawScopes: map[string]interface{}{ + "location": []string{"location-1"}, + }, + expect: func(t *testing.T, mean float64) { + assert := assert.New(t) + assert.Equal(mean, float64(conditionLocationWeight)) + }, + }, + { + name: "match according to idc", + conditions: map[string]string{ + "idc": "idc-1", + }, + rawScopes: map[string]interface{}{ + "idc": []string{"idc-1"}, + }, + expect: func(t *testing.T, mean float64) { + assert := assert.New(t) + assert.Equal(mean, float64(conditionIDCWeight)) + }, + }, + { + name: "match according to location and idc", + conditions: map[string]string{ + "location": "location-1", + "idc": "idc-1", + }, + rawScopes: map[string]interface{}{ + "location": []string{"location-1"}, + "idc": []string{"idc-1"}, + }, + expect: func(t *testing.T, mean float64) { + assert := assert.New(t) + assert.Equal(mean, float64(conditionLocationWeight+conditionIDCWeight)) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mean := calculateSchedulerClusterMean(tc.conditions, tc.rawScopes) + tc.expect(t, mean) + }) + } +} + +func TestCalculateConditionScore(t *testing.T) { + tests := []struct { + name string + value string + scope []string + expect func(t *testing.T, score float64) + }{ + { + name: "value is empty", + value: "", + scope: []string{"foo"}, + expect: func(t *testing.T, score float64) { + assert := assert.New(t) + assert.Equal(score, float64(0)) + }, + }, + { + name: "scope is empty", + value: "foo", + scope: []string{}, + expect: func(t *testing.T, score float64) { + assert := assert.New(t) + assert.Equal(score, float64(0)) + }, + }, + { + name: "match according to value", + value: "foo", + scope: []string{"foo"}, + expect: func(t *testing.T, score float64) { + assert := assert.New(t) + assert.Equal(score, float64(1)) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + score := calculateConditionScore(tc.value, tc.scope) + tc.expect(t, score) + }) + } +} diff --git a/manager/searcher/testdata/main.go b/manager/searcher/testdata/main.go new file mode 100644 index 000000000..880b47108 --- /dev/null +++ b/manager/searcher/testdata/main.go @@ -0,0 +1,52 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "fmt" + "os" + + "d7y.io/dragonfly/v2/internal/dfpath" + "d7y.io/dragonfly/v2/manager/model" + "d7y.io/dragonfly/v2/manager/searcher" +) + +func init() { + flag.StringVar(&dfpath.PluginsDir, "plugin-dir", ".", "") +} + +func main() { + flag.Parse() + + s, err := searcher.LoadPlugin() + if err != nil { + fmt.Printf("load plugin error: %s\n", err) + os.Exit(1) + } + + cluster, ok := s.FindSchedulerCluster([]model.SchedulerCluster{}, map[string]string{}) + if !ok { + fmt.Println("scheduler cluster not found") + os.Exit(1) + } + + if cluster.Name != "foo" { + fmt.Println("scheduler cluster name wrong") + os.Exit(1) + } +} diff --git a/manager/searcher/testdata/plugin/searcher.go b/manager/searcher/testdata/plugin/searcher.go new file mode 100644 index 000000000..5b400f374 --- /dev/null +++ b/manager/searcher/testdata/plugin/searcher.go @@ -0,0 +1,29 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import "d7y.io/dragonfly/v2/manager/model" + +type searcher struct{} + +func (s *searcher) FindSchedulerCluster(schedulerClusters []model.SchedulerCluster, conditions map[string]string) (model.SchedulerCluster, bool) { + return model.SchedulerCluster{Name: "foo"}, true +} + +func DragonflyPluginInit(option map[string]string) (interface{}, map[string]string, error) { + return &searcher{}, map[string]string{"type": "manager", "name": "searcher"}, nil +} diff --git a/manager/server/server.go b/manager/server/server.go index a9a85ede4..7524f3b39 100644 --- a/manager/server/server.go +++ b/manager/server/server.go @@ -24,6 +24,7 @@ import ( "d7y.io/dragonfly/v2/manager/cache" "d7y.io/dragonfly/v2/manager/config" "d7y.io/dragonfly/v2/manager/database" + "d7y.io/dragonfly/v2/manager/searcher" "d7y.io/dragonfly/v2/manager/service" "d7y.io/dragonfly/v2/pkg/rpc" "d7y.io/dragonfly/v2/pkg/rpc/manager" @@ -49,9 +50,12 @@ func New(cfg *config.Config) (*Server, error) { return nil, err } - // Initialize database + // Initialize cache cache := cache.New(cfg) + // Initialize searcher + searcher := searcher.New() + // Initialize REST service restService := service.NewREST( service.WithDatabase(db), @@ -62,6 +66,7 @@ func New(cfg *config.Config) (*Server, error) { grpcService := service.NewGRPC( service.GRPCWithDatabase(db), service.GRPCWithCache(cache), + service.GRPCWithSearcher(searcher), ) // Initialize router diff --git a/manager/service/scheduler_cluster.go b/manager/service/scheduler_cluster.go index ebc5e7668..abe329c44 100644 --- a/manager/service/scheduler_cluster.go +++ b/manager/service/scheduler_cluster.go @@ -11,6 +11,7 @@ func (s *rest) CreateSchedulerCluster(json types.CreateSchedulerClusterRequest) BIO: json.BIO, Config: json.Config, ClientConfig: json.ClientConfig, + Scopes: json.Scopes, IsDefault: json.IsDefault, } @@ -34,6 +35,7 @@ func (s *rest) CreateSchedulerClusterWithSecurityGroupDomain(json types.CreateSc BIO: json.BIO, Config: json.Config, ClientConfig: json.ClientConfig, + Scopes: json.Scopes, IsDefault: json.IsDefault, } @@ -59,6 +61,7 @@ func (s *rest) UpdateSchedulerCluster(id uint, json types.UpdateSchedulerCluster BIO: json.BIO, Config: json.Config, ClientConfig: json.ClientConfig, + Scopes: json.Scopes, IsDefault: json.IsDefault, }).Error; err != nil { return nil, err @@ -80,6 +83,7 @@ func (s *rest) UpdateSchedulerClusterWithSecurityGroupDomain(id uint, json types BIO: json.BIO, Config: json.Config, ClientConfig: json.ClientConfig, + Scopes: json.Scopes, IsDefault: json.IsDefault, } diff --git a/manager/service/service_grpc.go b/manager/service/service_grpc.go index 75d50aff4..f20504248 100644 --- a/manager/service/service_grpc.go +++ b/manager/service/service_grpc.go @@ -8,6 +8,7 @@ import ( "d7y.io/dragonfly/v2/manager/cache" "d7y.io/dragonfly/v2/manager/database" "d7y.io/dragonfly/v2/manager/model" + "d7y.io/dragonfly/v2/manager/searcher" "d7y.io/dragonfly/v2/pkg/rpc/manager" cachev8 "github.com/go-redis/cache/v8" "github.com/go-redis/redis/v8" @@ -23,12 +24,13 @@ type GRPC struct { rdb *redis.Client cache *cache.Cache manager.UnimplementedManagerServer + searcher searcher.Searcher } // Option is a functional option for rest type GRPCOption func(s *GRPC) -// WithDatabase set the database client +// GRPCWithDatabase set the database client func GRPCWithDatabase(database *database.Database) GRPCOption { return func(s *GRPC) { s.db = database.DB @@ -36,13 +38,20 @@ func GRPCWithDatabase(database *database.Database) GRPCOption { } } -// WithCache set the cache client +// GRPCWithCache set the cache client func GRPCWithCache(cache *cache.Cache) GRPCOption { return func(s *GRPC) { s.cache = cache } } +// GRPCWithSearcher set search client +func GRPCWithSearcher(searcher searcher.Searcher) GRPCOption { + return func(s *GRPC) { + s.searcher = searcher + } +} + // NewREST returns a new REST instence func NewGRPC(options ...GRPCOption) *GRPC { s := &GRPC{} @@ -433,9 +442,25 @@ func (s *GRPC) ListSchedulers(ctx context.Context, req *manager.ListSchedulersRe // Cache Miss logger.Infof("%s cache miss", cacheKey) + var schedulerClusters []model.SchedulerCluster + if err := s.db.Preload("SecurityGroup").Find(&schedulerClusters).Error; err != nil { + return nil, status.Error(codes.Unknown, err.Error()) + } + + // Search optimal scheduler cluster + schedulerCluster, ok := s.searcher.FindSchedulerCluster(schedulerClusters, req.HostInfo) + if !ok { + if err := s.db.Find(&schedulerCluster, &model.SchedulerCluster{ + IsDefault: true, + }).Error; err != nil { + return nil, status.Error(codes.Unknown, err.Error()) + } + } + schedulers := []model.Scheduler{} if err := s.db.Find(&schedulers, &model.Scheduler{ - Status: model.SchedulerStatusActive, + Status: model.SchedulerStatusActive, + SchedulerClusterID: &schedulerCluster.ID, }).Error; err != nil { return nil, status.Error(codes.Unknown, err.Error()) } diff --git a/manager/types/scheduler_cluster.go b/manager/types/scheduler_cluster.go index 70a40fc84..0b837e9ab 100644 --- a/manager/types/scheduler_cluster.go +++ b/manager/types/scheduler_cluster.go @@ -14,6 +14,7 @@ type CreateSchedulerClusterRequest struct { BIO string `json:"bio" binding:"omitempty"` Config map[string]interface{} `json:"config" binding:"required"` ClientConfig map[string]interface{} `json:"client_config" binding:"required"` + Scopes map[string]interface{} `json:"scopes" binding:"omitempty"` IsDefault bool `json:"is_default" binding:"omitempty"` SecurityGroupDomain string `json:"security_group_domain" binding:"omitempty"` } @@ -23,6 +24,7 @@ type UpdateSchedulerClusterRequest struct { BIO string `json:"bio" binding:"omitempty"` Config map[string]interface{} `json:"config" binding:"omitempty"` ClientConfig map[string]interface{} `json:"client_config" binding:"omitempty"` + Scopes map[string]interface{} `json:"scopes" binding:"omitempty"` IsDefault bool `json:"is_default" binding:"omitempty"` SecurityGroupDomain string `json:"security_group_domain" binding:"omitempty"` }