Add cron support
This commit is contained in:
Yinan Li
parent
3624ad80de
commit
eff4c59cb2
|
|
@ -3,15 +3,27 @@
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "cloud.google.com/go"
|
name = "cloud.google.com/go"
|
||||||
packages = ["compute/metadata","iam","internal","internal/optional","internal/version","storage"]
|
packages = [
|
||||||
|
"compute/metadata",
|
||||||
|
"iam",
|
||||||
|
"internal",
|
||||||
|
"internal/optional",
|
||||||
|
"internal/version",
|
||||||
|
"storage"
|
||||||
|
]
|
||||||
revision = "767c40d6a2e058483c25fa193e963a22da17236d"
|
revision = "767c40d6a2e058483c25fa193e963a22da17236d"
|
||||||
version = "v0.18.0"
|
version = "v0.18.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/Azure/go-autorest"
|
name = "github.com/Azure/go-autorest"
|
||||||
packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
|
packages = [
|
||||||
revision = "c2a68353555b68de3ee8455a4fd3e890a0ac6d99"
|
"autorest",
|
||||||
version = "v9.8.1"
|
"autorest/adal",
|
||||||
|
"autorest/azure",
|
||||||
|
"autorest/date"
|
||||||
|
]
|
||||||
|
revision = "fc3b03a2d2d1f43fad3007038bd16f044f870722"
|
||||||
|
version = "v9.10.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/PuerkitoBio/purell"
|
name = "github.com/PuerkitoBio/purell"
|
||||||
|
|
@ -34,26 +46,26 @@
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/dgrijalva/jwt-go"
|
name = "github.com/dgrijalva/jwt-go"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29"
|
revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
|
||||||
version = "v3.1.0"
|
version = "v3.2.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/docker/spdystream"
|
name = "github.com/docker/spdystream"
|
||||||
packages = [".","spdy"]
|
packages = [
|
||||||
|
".",
|
||||||
|
"spdy"
|
||||||
|
]
|
||||||
revision = "bc6354cbbc295e925e4c611ffe90c1f287ee54db"
|
revision = "bc6354cbbc295e925e4c611ffe90c1f287ee54db"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/emicklei/go-restful"
|
name = "github.com/emicklei/go-restful"
|
||||||
packages = [".","log"]
|
packages = [
|
||||||
revision = "2dd44038f0b95ae693b266c5f87593b5d2fdd78d"
|
".",
|
||||||
version = "v2.5.0"
|
"log"
|
||||||
|
]
|
||||||
[[projects]]
|
revision = "92cd0815dd1a028a6e69faee9757c7436238e252"
|
||||||
name = "github.com/emicklei/go-restful-swagger12"
|
version = "v2.6.1"
|
||||||
packages = ["."]
|
|
||||||
revision = "dcef7f55730566d41eae5db10e7d6981829720f6"
|
|
||||||
version = "1.0.1"
|
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/ghodss/yaml"
|
name = "github.com/ghodss/yaml"
|
||||||
|
|
@ -65,31 +77,34 @@
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/go-openapi/jsonpointer"
|
name = "github.com/go-openapi/jsonpointer"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "779f45308c19820f1a69e9a4cd965f496e0da10f"
|
revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/go-openapi/jsonreference"
|
name = "github.com/go-openapi/jsonreference"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "36d33bfe519efae5632669801b180bf1a245da3b"
|
revision = "3fb327e6747da3043567ee86abd02bb6376b6be2"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/go-openapi/spec"
|
name = "github.com/go-openapi/spec"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "fa03337d7da5735229ee8f5e9d5d0b996014b7f8"
|
revision = "bcff419492eeeb01f76e77d2ebc714dc97b607f5"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/go-openapi/swag"
|
name = "github.com/go-openapi/swag"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "84f4bee7c0a6db40e3166044c7983c1c32125429"
|
revision = "811b1089cde9dad18d4d0c2d09fbdbf28dbd27a5"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/gogo/protobuf"
|
name = "github.com/gogo/protobuf"
|
||||||
packages = ["proto","sortkeys"]
|
packages = [
|
||||||
revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
|
"proto",
|
||||||
version = "v0.5"
|
"sortkeys"
|
||||||
|
]
|
||||||
|
revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
|
||||||
|
version = "v1.0.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
|
|
@ -101,13 +116,20 @@
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/golang/groupcache"
|
name = "github.com/golang/groupcache"
|
||||||
packages = ["lru"]
|
packages = ["lru"]
|
||||||
revision = "84a468cf14b4376def5d68c722b139b881c450a4"
|
revision = "66deaeb636dff1ac7d938ce666d090556056a4b0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
|
||||||
name = "github.com/golang/protobuf"
|
name = "github.com/golang/protobuf"
|
||||||
packages = ["proto","protoc-gen-go/descriptor","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"]
|
packages = [
|
||||||
revision = "c65a0412e71e8b9b3bfd22925720d23c0f054237"
|
"proto",
|
||||||
|
"protoc-gen-go/descriptor",
|
||||||
|
"ptypes",
|
||||||
|
"ptypes/any",
|
||||||
|
"ptypes/duration",
|
||||||
|
"ptypes/timestamp"
|
||||||
|
]
|
||||||
|
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
|
||||||
|
version = "v1.0.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
|
|
@ -129,27 +151,45 @@
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/googleapis/gnostic"
|
name = "github.com/googleapis/gnostic"
|
||||||
packages = ["OpenAPIv2","compiler","extensions"]
|
packages = [
|
||||||
|
"OpenAPIv2",
|
||||||
|
"compiler",
|
||||||
|
"extensions"
|
||||||
|
]
|
||||||
revision = "ee43cbb60db7bd22502942cccbc39059117352ab"
|
revision = "ee43cbb60db7bd22502942cccbc39059117352ab"
|
||||||
version = "v0.1.0"
|
version = "v0.1.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/gophercloud/gophercloud"
|
name = "github.com/gophercloud/gophercloud"
|
||||||
packages = [".","openstack","openstack/identity/v2/tenants","openstack/identity/v2/tokens","openstack/identity/v3/tokens","openstack/utils","pagination"]
|
packages = [
|
||||||
revision = "4a3f5ae58624b68283375060dad06a214b05a32b"
|
".",
|
||||||
|
"openstack",
|
||||||
|
"openstack/identity/v2/tenants",
|
||||||
|
"openstack/identity/v2/tokens",
|
||||||
|
"openstack/identity/v3/tokens",
|
||||||
|
"openstack/utils",
|
||||||
|
"pagination"
|
||||||
|
]
|
||||||
|
revision = "b06207adc597d227dd874274305c96e49c3015e3"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/gregjones/httpcache"
|
name = "github.com/gregjones/httpcache"
|
||||||
packages = [".","diskcache"]
|
packages = [
|
||||||
revision = "2bcd89a1743fd4b373f7370ce8ddc14dfbd18229"
|
".",
|
||||||
|
"diskcache"
|
||||||
|
]
|
||||||
|
revision = "9cad4c3443a7200dd6400aef47183728de563a38"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/hashicorp/golang-lru"
|
name = "github.com/hashicorp/golang-lru"
|
||||||
packages = [".","simplelru"]
|
packages = [
|
||||||
revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6"
|
".",
|
||||||
|
"simplelru"
|
||||||
|
]
|
||||||
|
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
|
|
@ -160,8 +200,8 @@
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/imdario/mergo"
|
name = "github.com/imdario/mergo"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "163f41321a19dd09362d4c63cc2489db2015f1f4"
|
revision = "9d5f1277e9a8ed20c3684bda8fde67c05628518c"
|
||||||
version = "0.3.2"
|
version = "v0.3.4"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/inconshreveable/mousetrap"
|
name = "github.com/inconshreveable/mousetrap"
|
||||||
|
|
@ -172,8 +212,8 @@
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/json-iterator/go"
|
name = "github.com/json-iterator/go"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "f7279a603edee96fe7764d3de9c6ff8cf9970994"
|
revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4"
|
||||||
version = "1.0.4"
|
version = "1.1.3"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/juju/ratelimit"
|
name = "github.com/juju/ratelimit"
|
||||||
|
|
@ -184,8 +224,12 @@
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/mailru/easyjson"
|
name = "github.com/mailru/easyjson"
|
||||||
packages = ["buffer","jlexer","jwriter"]
|
packages = [
|
||||||
revision = "32fa128f234d041f196a9f3e0fea5ac9772c08e1"
|
"buffer",
|
||||||
|
"jlexer",
|
||||||
|
"jwriter"
|
||||||
|
]
|
||||||
|
revision = "8b799c424f57fa123fc63a99d6383bc6e4c02578"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/mattn/go-runewidth"
|
name = "github.com/mattn/go-runewidth"
|
||||||
|
|
@ -193,11 +237,23 @@
|
||||||
revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
|
revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
|
||||||
version = "v0.0.2"
|
version = "v0.0.2"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/modern-go/concurrent"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
|
||||||
|
version = "1.0.3"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/modern-go/reflect2"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f"
|
||||||
|
version = "1.0.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/olekukonko/tablewriter"
|
name = "github.com/olekukonko/tablewriter"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "96aac992fc8b1a4c83841a6c3e7178d20d989625"
|
revision = "b8a9be070da40449e501c3c4730a889e42d87a9e"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
|
|
@ -217,123 +273,409 @@
|
||||||
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
|
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
|
||||||
version = "v1.0.0"
|
version = "v1.0.0"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/robfig/cron"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "df38d32658d8788cd446ba74db4bb5375c4b0cb3"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/spf13/cobra"
|
name = "github.com/spf13/cobra"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
|
revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4"
|
||||||
version = "v0.0.1"
|
version = "v0.0.2"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/spf13/pflag"
|
name = "github.com/spf13/pflag"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
|
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
|
||||||
version = "v1.0.0"
|
version = "v1.0.1"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/stretchr/testify"
|
name = "github.com/stretchr/testify"
|
||||||
packages = ["assert"]
|
packages = ["assert"]
|
||||||
revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
|
revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
|
||||||
version = "v1.2.0"
|
version = "v1.2.1"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "go.opencensus.io"
|
||||||
|
packages = [
|
||||||
|
"exporter/stackdriver/propagation",
|
||||||
|
"internal",
|
||||||
|
"internal/tagencoding",
|
||||||
|
"plugin/ochttp",
|
||||||
|
"plugin/ochttp/propagation/b3",
|
||||||
|
"stats",
|
||||||
|
"stats/internal",
|
||||||
|
"stats/view",
|
||||||
|
"tag",
|
||||||
|
"trace",
|
||||||
|
"trace/internal",
|
||||||
|
"trace/propagation"
|
||||||
|
]
|
||||||
|
revision = "0095aec66ae14801c6711210f6f0716411cefdd3"
|
||||||
|
version = "v0.8.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/crypto"
|
name = "golang.org/x/crypto"
|
||||||
packages = ["ssh/terminal"]
|
packages = ["ssh/terminal"]
|
||||||
revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b"
|
revision = "d6449816ce06963d9d136eee5a56fca5b0616e7e"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/net"
|
name = "golang.org/x/net"
|
||||||
packages = ["context","context/ctxhttp","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
|
packages = [
|
||||||
revision = "0ed95abb35c445290478a5348a7b38bb154135fd"
|
"context",
|
||||||
|
"context/ctxhttp",
|
||||||
|
"http/httpguts",
|
||||||
|
"http2",
|
||||||
|
"http2/hpack",
|
||||||
|
"idna",
|
||||||
|
"internal/timeseries",
|
||||||
|
"lex/httplex",
|
||||||
|
"trace"
|
||||||
|
]
|
||||||
|
revision = "8d16fa6dc9a85c1cd3ed24ad08ff21cf94f10888"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/oauth2"
|
name = "golang.org/x/oauth2"
|
||||||
packages = [".","google","internal","jws","jwt"]
|
packages = [
|
||||||
revision = "b28fcf2b08a19742b43084fb40ab78ac6c3d8067"
|
".",
|
||||||
|
"google",
|
||||||
|
"internal",
|
||||||
|
"jws",
|
||||||
|
"jwt"
|
||||||
|
]
|
||||||
|
revision = "6881fee410a5daf86371371f9ad451b95e168b71"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/sys"
|
name = "golang.org/x/sys"
|
||||||
packages = ["unix","windows"]
|
packages = [
|
||||||
revision = "03467258950d845cd1877eab69461b98e8c09219"
|
"unix",
|
||||||
|
"windows"
|
||||||
|
]
|
||||||
|
revision = "b126b21c05a91c856b027c16779c12e3bf236954"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
|
||||||
name = "golang.org/x/text"
|
name = "golang.org/x/text"
|
||||||
packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable","width"]
|
packages = [
|
||||||
revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
|
"collate",
|
||||||
|
"collate/build",
|
||||||
|
"internal/colltab",
|
||||||
|
"internal/gen",
|
||||||
|
"internal/tag",
|
||||||
|
"internal/triegen",
|
||||||
|
"internal/ucd",
|
||||||
|
"language",
|
||||||
|
"secure/bidirule",
|
||||||
|
"transform",
|
||||||
|
"unicode/bidi",
|
||||||
|
"unicode/cldr",
|
||||||
|
"unicode/norm",
|
||||||
|
"unicode/rangetable",
|
||||||
|
"width"
|
||||||
|
]
|
||||||
|
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
|
||||||
|
version = "v0.3.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "google.golang.org/api"
|
name = "google.golang.org/api"
|
||||||
packages = ["gensupport","googleapi","googleapi/internal/uritemplates","googleapi/transport","internal","iterator","option","storage/v1","transport/http"]
|
packages = [
|
||||||
revision = "a27ddaf5ce0ce688be462071a9e55566e60bb59e"
|
"gensupport",
|
||||||
|
"googleapi",
|
||||||
|
"googleapi/internal/uritemplates",
|
||||||
|
"googleapi/transport",
|
||||||
|
"internal",
|
||||||
|
"iterator",
|
||||||
|
"option",
|
||||||
|
"storage/v1",
|
||||||
|
"transport/http"
|
||||||
|
]
|
||||||
|
revision = "9c79deebf7496e355d7e95d82d4af1fe4e769b2f"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "google.golang.org/appengine"
|
name = "google.golang.org/appengine"
|
||||||
packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
|
packages = [
|
||||||
|
".",
|
||||||
|
"internal",
|
||||||
|
"internal/app_identity",
|
||||||
|
"internal/base",
|
||||||
|
"internal/datastore",
|
||||||
|
"internal/log",
|
||||||
|
"internal/modules",
|
||||||
|
"internal/remote_api",
|
||||||
|
"internal/urlfetch",
|
||||||
|
"urlfetch"
|
||||||
|
]
|
||||||
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
|
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
|
||||||
version = "v1.0.0"
|
version = "v1.0.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "google.golang.org/genproto"
|
name = "google.golang.org/genproto"
|
||||||
packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/rpc/status"]
|
packages = [
|
||||||
revision = "4eb30f4778eed4c258ba66527a0d4f9ec8a36c45"
|
"googleapis/api/annotations",
|
||||||
|
"googleapis/iam/v1",
|
||||||
|
"googleapis/rpc/status"
|
||||||
|
]
|
||||||
|
revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "google.golang.org/grpc"
|
name = "google.golang.org/grpc"
|
||||||
packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
|
packages = [
|
||||||
revision = "6b51017f791ae1cfbec89c52efdf444b13b550ef"
|
".",
|
||||||
version = "v1.9.2"
|
"balancer",
|
||||||
|
"balancer/base",
|
||||||
|
"balancer/roundrobin",
|
||||||
|
"codes",
|
||||||
|
"connectivity",
|
||||||
|
"credentials",
|
||||||
|
"encoding",
|
||||||
|
"encoding/proto",
|
||||||
|
"grpclb/grpc_lb_v1/messages",
|
||||||
|
"grpclog",
|
||||||
|
"internal",
|
||||||
|
"keepalive",
|
||||||
|
"metadata",
|
||||||
|
"naming",
|
||||||
|
"peer",
|
||||||
|
"resolver",
|
||||||
|
"resolver/dns",
|
||||||
|
"resolver/passthrough",
|
||||||
|
"stats",
|
||||||
|
"status",
|
||||||
|
"tap",
|
||||||
|
"transport"
|
||||||
|
]
|
||||||
|
revision = "d11072e7ca9811b1100b80ca0269ac831f06d024"
|
||||||
|
version = "v1.11.3"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "gopkg.in/inf.v0"
|
name = "gopkg.in/inf.v0"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4"
|
revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
|
||||||
version = "v0.9.0"
|
version = "v0.9.1"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "v2"
|
|
||||||
name = "gopkg.in/yaml.v2"
|
name = "gopkg.in/yaml.v2"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
|
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
|
||||||
|
version = "v2.2.1"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
|
branch = "release-1.9"
|
||||||
name = "k8s.io/api"
|
name = "k8s.io/api"
|
||||||
packages = ["admissionregistration/v1alpha1","apps/v1beta1","apps/v1beta2","authentication/v1","authentication/v1beta1","authorization/v1","authorization/v1beta1","autoscaling/v1","autoscaling/v2beta1","batch/v1","batch/v1beta1","batch/v2alpha1","certificates/v1beta1","core/v1","extensions/v1beta1","networking/v1","policy/v1beta1","rbac/v1","rbac/v1alpha1","rbac/v1beta1","scheduling/v1alpha1","settings/v1alpha1","storage/v1","storage/v1beta1"]
|
packages = [
|
||||||
revision = "4df58c811fe2e65feb879227b2b245e4dc26e7ad"
|
"admissionregistration/v1alpha1",
|
||||||
version = "kubernetes-1.8.2"
|
"admissionregistration/v1beta1",
|
||||||
|
"apps/v1",
|
||||||
|
"apps/v1beta1",
|
||||||
|
"apps/v1beta2",
|
||||||
|
"authentication/v1",
|
||||||
|
"authentication/v1beta1",
|
||||||
|
"authorization/v1",
|
||||||
|
"authorization/v1beta1",
|
||||||
|
"autoscaling/v1",
|
||||||
|
"autoscaling/v2beta1",
|
||||||
|
"batch/v1",
|
||||||
|
"batch/v1beta1",
|
||||||
|
"batch/v2alpha1",
|
||||||
|
"certificates/v1beta1",
|
||||||
|
"core/v1",
|
||||||
|
"events/v1beta1",
|
||||||
|
"extensions/v1beta1",
|
||||||
|
"networking/v1",
|
||||||
|
"policy/v1beta1",
|
||||||
|
"rbac/v1",
|
||||||
|
"rbac/v1alpha1",
|
||||||
|
"rbac/v1beta1",
|
||||||
|
"scheduling/v1alpha1",
|
||||||
|
"settings/v1alpha1",
|
||||||
|
"storage/v1",
|
||||||
|
"storage/v1alpha1",
|
||||||
|
"storage/v1beta1"
|
||||||
|
]
|
||||||
|
revision = "53400f2739169693b7f1f64cf856f6d3d780d3db"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
|
branch = "release-1.9"
|
||||||
name = "k8s.io/apiextensions-apiserver"
|
name = "k8s.io/apiextensions-apiserver"
|
||||||
packages = ["pkg/apis/apiextensions","pkg/apis/apiextensions/v1beta1","pkg/client/clientset/clientset","pkg/client/clientset/clientset/fake","pkg/client/clientset/clientset/scheme","pkg/client/clientset/clientset/typed/apiextensions/v1beta1","pkg/client/clientset/clientset/typed/apiextensions/v1beta1/fake"]
|
packages = [
|
||||||
revision = "e509bb64fe1116e12a32273a2032426aa1a5fd26"
|
"pkg/apis/apiextensions",
|
||||||
version = "kubernetes-1.8.2"
|
"pkg/apis/apiextensions/v1beta1",
|
||||||
|
"pkg/client/clientset/clientset",
|
||||||
|
"pkg/client/clientset/clientset/fake",
|
||||||
|
"pkg/client/clientset/clientset/scheme",
|
||||||
|
"pkg/client/clientset/clientset/typed/apiextensions/v1beta1",
|
||||||
|
"pkg/client/clientset/clientset/typed/apiextensions/v1beta1/fake"
|
||||||
|
]
|
||||||
|
revision = "9c19edfd52157adc354e1b72596534b47f23cb44"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
|
branch = "release-1.9"
|
||||||
name = "k8s.io/apimachinery"
|
name = "k8s.io/apimachinery"
|
||||||
packages = ["pkg/api/equality","pkg/api/errors","pkg/api/meta","pkg/api/resource","pkg/apis/meta/internalversion","pkg/apis/meta/v1","pkg/apis/meta/v1/unstructured","pkg/apis/meta/v1alpha1","pkg/conversion","pkg/conversion/queryparams","pkg/conversion/unstructured","pkg/fields","pkg/labels","pkg/runtime","pkg/runtime/schema","pkg/runtime/serializer","pkg/runtime/serializer/json","pkg/runtime/serializer/protobuf","pkg/runtime/serializer/recognizer","pkg/runtime/serializer/streaming","pkg/runtime/serializer/versioning","pkg/selection","pkg/types","pkg/util/cache","pkg/util/clock","pkg/util/diff","pkg/util/errors","pkg/util/framer","pkg/util/httpstream","pkg/util/httpstream/spdy","pkg/util/intstr","pkg/util/json","pkg/util/mergepatch","pkg/util/net","pkg/util/runtime","pkg/util/sets","pkg/util/strategicpatch","pkg/util/validation","pkg/util/validation/field","pkg/util/wait","pkg/util/yaml","pkg/version","pkg/watch","third_party/forked/golang/json","third_party/forked/golang/netutil","third_party/forked/golang/reflect"]
|
packages = [
|
||||||
revision = "019ae5ada31de202164b118aee88ee2d14075c31"
|
"pkg/api/errors",
|
||||||
version = "kubernetes-1.8.0"
|
"pkg/api/meta",
|
||||||
|
"pkg/api/resource",
|
||||||
|
"pkg/apis/meta/internalversion",
|
||||||
|
"pkg/apis/meta/v1",
|
||||||
|
"pkg/apis/meta/v1/unstructured",
|
||||||
|
"pkg/apis/meta/v1alpha1",
|
||||||
|
"pkg/conversion",
|
||||||
|
"pkg/conversion/queryparams",
|
||||||
|
"pkg/fields",
|
||||||
|
"pkg/labels",
|
||||||
|
"pkg/runtime",
|
||||||
|
"pkg/runtime/schema",
|
||||||
|
"pkg/runtime/serializer",
|
||||||
|
"pkg/runtime/serializer/json",
|
||||||
|
"pkg/runtime/serializer/protobuf",
|
||||||
|
"pkg/runtime/serializer/recognizer",
|
||||||
|
"pkg/runtime/serializer/streaming",
|
||||||
|
"pkg/runtime/serializer/versioning",
|
||||||
|
"pkg/selection",
|
||||||
|
"pkg/types",
|
||||||
|
"pkg/util/cache",
|
||||||
|
"pkg/util/clock",
|
||||||
|
"pkg/util/diff",
|
||||||
|
"pkg/util/errors",
|
||||||
|
"pkg/util/framer",
|
||||||
|
"pkg/util/httpstream",
|
||||||
|
"pkg/util/httpstream/spdy",
|
||||||
|
"pkg/util/intstr",
|
||||||
|
"pkg/util/json",
|
||||||
|
"pkg/util/mergepatch",
|
||||||
|
"pkg/util/net",
|
||||||
|
"pkg/util/runtime",
|
||||||
|
"pkg/util/sets",
|
||||||
|
"pkg/util/strategicpatch",
|
||||||
|
"pkg/util/validation",
|
||||||
|
"pkg/util/validation/field",
|
||||||
|
"pkg/util/wait",
|
||||||
|
"pkg/util/yaml",
|
||||||
|
"pkg/version",
|
||||||
|
"pkg/watch",
|
||||||
|
"third_party/forked/golang/json",
|
||||||
|
"third_party/forked/golang/netutil",
|
||||||
|
"third_party/forked/golang/reflect"
|
||||||
|
]
|
||||||
|
revision = "4a1945a9cfdfa104d202059c7e64250618d8d009"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
|
branch = "release-6.0"
|
||||||
name = "k8s.io/client-go"
|
name = "k8s.io/client-go"
|
||||||
packages = ["discovery","discovery/fake","kubernetes","kubernetes/fake","kubernetes/scheme","kubernetes/typed/admissionregistration/v1alpha1","kubernetes/typed/admissionregistration/v1alpha1/fake","kubernetes/typed/apps/v1beta1","kubernetes/typed/apps/v1beta1/fake","kubernetes/typed/apps/v1beta2","kubernetes/typed/apps/v1beta2/fake","kubernetes/typed/authentication/v1","kubernetes/typed/authentication/v1/fake","kubernetes/typed/authentication/v1beta1","kubernetes/typed/authentication/v1beta1/fake","kubernetes/typed/authorization/v1","kubernetes/typed/authorization/v1/fake","kubernetes/typed/authorization/v1beta1","kubernetes/typed/authorization/v1beta1/fake","kubernetes/typed/autoscaling/v1","kubernetes/typed/autoscaling/v1/fake","kubernetes/typed/autoscaling/v2beta1","kubernetes/typed/autoscaling/v2beta1/fake","kubernetes/typed/batch/v1","kubernetes/typed/batch/v1/fake","kubernetes/typed/batch/v1beta1","kubernetes/typed/batch/v1beta1/fake","kubernetes/typed/batch/v2alpha1","kubernetes/typed/batch/v2alpha1/fake","kubernetes/typed/certificates/v1beta1","kubernetes/typed/certificates/v1beta1/fake","kubernetes/typed/core/v1","kubernetes/typed/core/v1/fake","kubernetes/typed/extensions/v1beta1","kubernetes/typed/extensions/v1beta1/fake","kubernetes/typed/networking/v1","kubernetes/typed/networking/v1/fake","kubernetes/typed/policy/v1beta1","kubernetes/typed/policy/v1beta1/fake","kubernetes/typed/rbac/v1","kubernetes/typed/rbac/v1/fake","kubernetes/typed/rbac/v1alpha1","kubernetes/typed/rbac/v1alpha1/fake","kubernetes/typed/rbac/v1beta1","kubernetes/typed/rbac/v1beta1/fake","kubernetes/typed/scheduling/v1alpha1","kubernetes/typed/scheduling/v1alpha1/fake","kubernetes/typed/settings/v1alpha1","kubernetes/typed/settings/v1alpha1/fake","kubernetes/typed/storage/v1","kubernetes/typed/storage/v1/fake","kubernetes/typed/storage/v1beta1","kubernetes/typed/storage/v1beta1/fake","pkg/version","plugin/pkg/client/auth","plugin/pkg/client/auth/azure","plugin/pkg/client/auth/gcp","plugin/pkg/client/auth/oidc","plugin/pkg/client/auth/openstack","rest","rest/watch","testing","third_party/forked/golang/template","tools/auth","tools/cache","tools/clientcmd","tools/clientcmd/api","tools/clientcmd/api/latest","tools/clientcmd/api/v1","tools/metrics","tools/pager","tools/portforward","tools/record","tools/reference","transport","transport/spdy","util/cert","util/flowcontrol","util/homedir","util/integer","util/jsonpath","util/workqueue"]
|
packages = [
|
||||||
revision = "35ccd4336052e7d73018b1382413534936f34eee"
|
"discovery",
|
||||||
version = "kubernetes-1.8.2"
|
"discovery/fake",
|
||||||
|
"kubernetes",
|
||||||
|
"kubernetes/fake",
|
||||||
|
"kubernetes/scheme",
|
||||||
|
"kubernetes/typed/admissionregistration/v1alpha1",
|
||||||
|
"kubernetes/typed/admissionregistration/v1alpha1/fake",
|
||||||
|
"kubernetes/typed/admissionregistration/v1beta1",
|
||||||
|
"kubernetes/typed/admissionregistration/v1beta1/fake",
|
||||||
|
"kubernetes/typed/apps/v1",
|
||||||
|
"kubernetes/typed/apps/v1/fake",
|
||||||
|
"kubernetes/typed/apps/v1beta1",
|
||||||
|
"kubernetes/typed/apps/v1beta1/fake",
|
||||||
|
"kubernetes/typed/apps/v1beta2",
|
||||||
|
"kubernetes/typed/apps/v1beta2/fake",
|
||||||
|
"kubernetes/typed/authentication/v1",
|
||||||
|
"kubernetes/typed/authentication/v1/fake",
|
||||||
|
"kubernetes/typed/authentication/v1beta1",
|
||||||
|
"kubernetes/typed/authentication/v1beta1/fake",
|
||||||
|
"kubernetes/typed/authorization/v1",
|
||||||
|
"kubernetes/typed/authorization/v1/fake",
|
||||||
|
"kubernetes/typed/authorization/v1beta1",
|
||||||
|
"kubernetes/typed/authorization/v1beta1/fake",
|
||||||
|
"kubernetes/typed/autoscaling/v1",
|
||||||
|
"kubernetes/typed/autoscaling/v1/fake",
|
||||||
|
"kubernetes/typed/autoscaling/v2beta1",
|
||||||
|
"kubernetes/typed/autoscaling/v2beta1/fake",
|
||||||
|
"kubernetes/typed/batch/v1",
|
||||||
|
"kubernetes/typed/batch/v1/fake",
|
||||||
|
"kubernetes/typed/batch/v1beta1",
|
||||||
|
"kubernetes/typed/batch/v1beta1/fake",
|
||||||
|
"kubernetes/typed/batch/v2alpha1",
|
||||||
|
"kubernetes/typed/batch/v2alpha1/fake",
|
||||||
|
"kubernetes/typed/certificates/v1beta1",
|
||||||
|
"kubernetes/typed/certificates/v1beta1/fake",
|
||||||
|
"kubernetes/typed/core/v1",
|
||||||
|
"kubernetes/typed/core/v1/fake",
|
||||||
|
"kubernetes/typed/events/v1beta1",
|
||||||
|
"kubernetes/typed/events/v1beta1/fake",
|
||||||
|
"kubernetes/typed/extensions/v1beta1",
|
||||||
|
"kubernetes/typed/extensions/v1beta1/fake",
|
||||||
|
"kubernetes/typed/networking/v1",
|
||||||
|
"kubernetes/typed/networking/v1/fake",
|
||||||
|
"kubernetes/typed/policy/v1beta1",
|
||||||
|
"kubernetes/typed/policy/v1beta1/fake",
|
||||||
|
"kubernetes/typed/rbac/v1",
|
||||||
|
"kubernetes/typed/rbac/v1/fake",
|
||||||
|
"kubernetes/typed/rbac/v1alpha1",
|
||||||
|
"kubernetes/typed/rbac/v1alpha1/fake",
|
||||||
|
"kubernetes/typed/rbac/v1beta1",
|
||||||
|
"kubernetes/typed/rbac/v1beta1/fake",
|
||||||
|
"kubernetes/typed/scheduling/v1alpha1",
|
||||||
|
"kubernetes/typed/scheduling/v1alpha1/fake",
|
||||||
|
"kubernetes/typed/settings/v1alpha1",
|
||||||
|
"kubernetes/typed/settings/v1alpha1/fake",
|
||||||
|
"kubernetes/typed/storage/v1",
|
||||||
|
"kubernetes/typed/storage/v1/fake",
|
||||||
|
"kubernetes/typed/storage/v1alpha1",
|
||||||
|
"kubernetes/typed/storage/v1alpha1/fake",
|
||||||
|
"kubernetes/typed/storage/v1beta1",
|
||||||
|
"kubernetes/typed/storage/v1beta1/fake",
|
||||||
|
"pkg/version",
|
||||||
|
"plugin/pkg/client/auth",
|
||||||
|
"plugin/pkg/client/auth/azure",
|
||||||
|
"plugin/pkg/client/auth/gcp",
|
||||||
|
"plugin/pkg/client/auth/oidc",
|
||||||
|
"plugin/pkg/client/auth/openstack",
|
||||||
|
"rest",
|
||||||
|
"rest/watch",
|
||||||
|
"testing",
|
||||||
|
"third_party/forked/golang/template",
|
||||||
|
"tools/auth",
|
||||||
|
"tools/cache",
|
||||||
|
"tools/clientcmd",
|
||||||
|
"tools/clientcmd/api",
|
||||||
|
"tools/clientcmd/api/latest",
|
||||||
|
"tools/clientcmd/api/v1",
|
||||||
|
"tools/metrics",
|
||||||
|
"tools/pager",
|
||||||
|
"tools/portforward",
|
||||||
|
"tools/record",
|
||||||
|
"tools/reference",
|
||||||
|
"transport",
|
||||||
|
"transport/spdy",
|
||||||
|
"util/buffer",
|
||||||
|
"util/cert",
|
||||||
|
"util/flowcontrol",
|
||||||
|
"util/homedir",
|
||||||
|
"util/integer",
|
||||||
|
"util/jsonpath",
|
||||||
|
"util/retry",
|
||||||
|
"util/workqueue"
|
||||||
|
]
|
||||||
|
revision = "65b43df093d1d129e9608582d98ac9bfbba7e486"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "k8s.io/kube-openapi"
|
name = "k8s.io/kube-openapi"
|
||||||
packages = ["pkg/common"]
|
packages = [
|
||||||
revision = "a07b7bbb58e7fdc5144f8d7046331d29fc9ad3b3"
|
"pkg/common",
|
||||||
|
"pkg/util/proto"
|
||||||
|
]
|
||||||
|
revision = "f442ecb314a3679150c272e2b9713d8deed5955d"
|
||||||
|
|
||||||
[solve-meta]
|
[solve-meta]
|
||||||
analyzer-name = "dep"
|
analyzer-name = "dep"
|
||||||
analyzer-version = 1
|
analyzer-version = 1
|
||||||
inputs-digest = "b38ab52c44d1c54760fca3559177d6b68d117b9b37bc950465d0ab3e4b51b5e7"
|
inputs-digest = "3ab80822024d0dfcc1bb6ef3ae7c320a901d43b06f31fd1594704e6d836185af"
|
||||||
solver-name = "gps-cdcl"
|
solver-name = "gps-cdcl"
|
||||||
solver-version = 1
|
solver-version = 1
|
||||||
|
|
|
||||||
16
Gopkg.toml
16
Gopkg.toml
|
|
@ -55,21 +55,29 @@
|
||||||
name = "golang.org/x/net"
|
name = "golang.org/x/net"
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
|
branch = "release-1.9"
|
||||||
name = "k8s.io/api"
|
name = "k8s.io/api"
|
||||||
version = "kubernetes-1.8.2"
|
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
|
branch = "release-1.9"
|
||||||
name = "k8s.io/apiextensions-apiserver"
|
name = "k8s.io/apiextensions-apiserver"
|
||||||
version = "kubernetes-1.8.2"
|
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
|
branch = "release-1.9"
|
||||||
name = "k8s.io/apimachinery"
|
name = "k8s.io/apimachinery"
|
||||||
version = "kubernetes-1.8.2"
|
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
|
branch = "release-6.0"
|
||||||
name = "k8s.io/client-go"
|
name = "k8s.io/client-go"
|
||||||
version = "kubernetes-1.8.2"
|
|
||||||
|
[[override]]
|
||||||
|
name = "github.com/Azure/go-autorest"
|
||||||
|
version = "v9.1.0"
|
||||||
|
|
||||||
[[constraint]]
|
[[constraint]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/olekukonko/tablewriter"
|
name = "github.com/olekukonko/tablewriter"
|
||||||
|
|
||||||
|
[[constraint]]
|
||||||
|
name = "github.com/robfig/cron"
|
||||||
|
revision = "df38d32658d8788cd446ba74db4bb5375c4b0cb3"
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,14 @@ The number of worker threads to use in the three places are controlled using com
|
||||||
`-initializer-threads` and `-submission-threads`, respectively. The default values for the flags are 10, 10, and 3,
|
`-initializer-threads` and `-submission-threads`, respectively. The default values for the flags are 10, 10, and 3,
|
||||||
respectively.
|
respectively.
|
||||||
|
|
||||||
|
Spark Operator enables cache resynchronization so periodically the informers used by the operator will re-list existing
|
||||||
|
objects it manages and re-trigger resource events. The resynchronization interval in seconds can be configured using the
|
||||||
|
flag `-resync-interval`, with a default value of 30 seconds.
|
||||||
|
|
||||||
|
By default, Spark Operator will install the
|
||||||
|
[CustomResourceDefinitions](https://kubernetes.io/docs/tasks/access-kubernetes-api/extend-api-custom-resource-definitions/)
|
||||||
|
for the custom resources it managers. This can be disabled by setting the flag `-install-crds=false.`.
|
||||||
|
|
||||||
The initializer is an **optional** component and can be enabled or disabled using the `-enable-initializer` flag, which
|
The initializer is an **optional** component and can be enabled or disabled using the `-enable-initializer` flag, which
|
||||||
defaults to `true`. Since the initializer is an alpha feature, it won't function in Kubernetes clusters without alpha
|
defaults to `true`. Since the initializer is an alpha feature, it won't function in Kubernetes clusters without alpha
|
||||||
features enabled. In this case, it can be disabled by adding the argument `-enable-initializer=false` to
|
features enabled. In this case, it can be disabled by adding the argument `-enable-initializer=false` to
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
#
|
||||||
|
# Copyright 2018 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
apiVersion: "sparkoperator.k8s.io/v1alpha1"
|
||||||
|
kind: ScheduledSparkApplication
|
||||||
|
metadata:
|
||||||
|
name: spark-pi-scheduled
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
schedule: "@every 5m"
|
||||||
|
concurrencyPolicy: Allow
|
||||||
|
template:
|
||||||
|
type: Scala
|
||||||
|
mode: cluster
|
||||||
|
image: "gcr.io/ynli-k8s/spark:v2.3.0"
|
||||||
|
mainClass: org.apache.spark.examples.SparkPi
|
||||||
|
mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar"
|
||||||
|
driver:
|
||||||
|
cores: 0.1
|
||||||
|
coreLimit: "200m"
|
||||||
|
memory: "512m"
|
||||||
|
labels:
|
||||||
|
version: 2.3.0
|
||||||
|
serviceAccount: spark
|
||||||
|
executor:
|
||||||
|
cores: 1
|
||||||
|
instances: 1
|
||||||
|
memory: "512m"
|
||||||
|
labels:
|
||||||
|
version: 2.3.0
|
||||||
|
restartPolicy: Never
|
||||||
38
main.go
38
main.go
|
|
@ -29,6 +29,7 @@ import (
|
||||||
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||||
"k8s.io/apimachinery/pkg/api/errors"
|
"k8s.io/apimachinery/pkg/api/errors"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/clock"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
|
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
|
||||||
"k8s.io/client-go/rest"
|
"k8s.io/client-go/rest"
|
||||||
|
|
@ -36,7 +37,10 @@ import (
|
||||||
|
|
||||||
crdclientset "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned"
|
crdclientset "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned"
|
||||||
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
||||||
"k8s.io/spark-on-k8s-operator/pkg/controller"
|
"k8s.io/spark-on-k8s-operator/pkg/controller/scheduledsparkapplication"
|
||||||
|
"k8s.io/spark-on-k8s-operator/pkg/controller/sparkapplication"
|
||||||
|
ssacrd "k8s.io/spark-on-k8s-operator/pkg/crd/scheduledsparkapplication"
|
||||||
|
sacrd "k8s.io/spark-on-k8s-operator/pkg/crd/sparkapplication"
|
||||||
"k8s.io/spark-on-k8s-operator/pkg/initializer"
|
"k8s.io/spark-on-k8s-operator/pkg/initializer"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -47,12 +51,14 @@ var (
|
||||||
"out-of-cluster.")
|
"out-of-cluster.")
|
||||||
enableInitializer = flag.Bool("enable-initializer", true, "Whether to enable the "+
|
enableInitializer = flag.Bool("enable-initializer", true, "Whether to enable the "+
|
||||||
"Spark pod initializer.")
|
"Spark pod initializer.")
|
||||||
|
installCRDs = flag.Bool("install-crds", true, "Whether to install CRDs")
|
||||||
initializerThreads = flag.Int("initializer-threads", 10, "Number of worker threads "+
|
initializerThreads = flag.Int("initializer-threads", 10, "Number of worker threads "+
|
||||||
"used by the Spark Pod initializer (if it's enabled).")
|
"used by the Spark Pod initializer (if it's enabled).")
|
||||||
controllerThreads = flag.Int("controller-threads", 10, "Number of worker threads "+
|
controllerThreads = flag.Int("controller-threads", 10, "Number of worker threads "+
|
||||||
"used by the SparkApplication controller.")
|
"used by the SparkApplication controller.")
|
||||||
submissionRunnerThreads = flag.Int("submission-threads", 3, "Number of worker threads "+
|
submissionRunnerThreads = flag.Int("submission-threads", 3, "Number of worker threads "+
|
||||||
"used by the SparkApplication submission runner.")
|
"used by the SparkApplication submission runner.")
|
||||||
|
resyncInterval = flag.Int("resync-interval", 30, "Informer resync interval in seconds")
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
|
@ -86,16 +92,35 @@ func main() {
|
||||||
glog.Fatal(err)
|
glog.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if *installCRDs {
|
||||||
|
glog.Infof("Creating CustomResourceDefinition %s", sacrd.FullName)
|
||||||
|
err = sacrd.CreateCRD(apiExtensionsClient)
|
||||||
|
if err != nil {
|
||||||
|
glog.Fatalf("failed to create CustomResourceDefinition %s: %v", sacrd.FullName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
glog.Infof("Creating CustomResourceDefinition %s", ssacrd.FullName)
|
||||||
|
err = ssacrd.CreateCRD(apiExtensionsClient)
|
||||||
|
if err != nil {
|
||||||
|
glog.Fatalf("failed to create CustomResourceDefinition %s: %v", ssacrd.FullName, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
factory := crdinformers.NewSharedInformerFactory(
|
factory := crdinformers.NewSharedInformerFactory(
|
||||||
crdClient,
|
crdClient,
|
||||||
// resyncPeriod. Every resyncPeriod, all resources in the cache will re-trigger events.
|
// resyncPeriod. Every resyncPeriod, all resources in the cache will re-trigger events.
|
||||||
300*time.Second)
|
time.Duration(*resyncInterval)*time.Second)
|
||||||
sparkApplicationController := controller.New(crdClient, kubeClient, apiExtensionsClient, factory,
|
applicationController := sparkapplication.NewController(
|
||||||
*submissionRunnerThreads)
|
crdClient, kubeClient, apiExtensionsClient, factory, *submissionRunnerThreads)
|
||||||
|
scheduledApplicationController := scheduledsparkapplication.NewController(
|
||||||
|
crdClient, kubeClient, apiExtensionsClient, factory, clock.RealClock{})
|
||||||
|
|
||||||
// Start the informer factory that in turn starts the informer.
|
// Start the informer factory that in turn starts the informer.
|
||||||
go factory.Start(stopCh)
|
go factory.Start(stopCh)
|
||||||
if err = sparkApplicationController.Start(*controllerThreads, stopCh); err != nil {
|
if err = applicationController.Start(*controllerThreads, stopCh); err != nil {
|
||||||
|
glog.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = scheduledApplicationController.Start(*controllerThreads, stopCh); err != nil {
|
||||||
glog.Fatal(err)
|
glog.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -115,7 +140,8 @@ func main() {
|
||||||
close(stopCh)
|
close(stopCh)
|
||||||
|
|
||||||
glog.Info("Shutting down the Spark operator")
|
glog.Info("Shutting down the Spark operator")
|
||||||
sparkApplicationController.Stop()
|
applicationController.Stop()
|
||||||
|
scheduledApplicationController.Stop()
|
||||||
if *enableInitializer {
|
if *enableInitializer {
|
||||||
sparkPodInitializer.Stop()
|
sparkPodInitializer.Stop()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
|
||||||
scheme.AddKnownTypes(SchemeGroupVersion,
|
scheme.AddKnownTypes(SchemeGroupVersion,
|
||||||
&SparkApplication{},
|
&SparkApplication{},
|
||||||
&SparkApplicationList{},
|
&SparkApplicationList{},
|
||||||
|
&ScheduledSparkApplication{},
|
||||||
|
&ScheduledSparkApplicationList{},
|
||||||
)
|
)
|
||||||
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
|
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
|
||||||
return nil
|
return nil
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,74 @@ const (
|
||||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||||
// +k8s:defaulter-gen=true
|
// +k8s:defaulter-gen=true
|
||||||
|
|
||||||
|
type ScheduledSparkApplication struct {
|
||||||
|
metav1.TypeMeta `json:",inline"`
|
||||||
|
metav1.ObjectMeta `json:"metadata"`
|
||||||
|
Spec ScheduledSparkApplicationSpec `json:"spec"`
|
||||||
|
Status ScheduledSparkApplicationStatus `json:"status,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ConcurrencyPolicy string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ConcurrencyAllow allows SparkApplications to run concurrently.
|
||||||
|
ConcurrencyAllow ConcurrencyPolicy = "Allow"
|
||||||
|
// ConcurrencyForbid forbids concurrent runs of SparkApplications, skipping the next run if the previous
|
||||||
|
// one hasn't finished yet.
|
||||||
|
ConcurrencyForbid ConcurrencyPolicy = "Forbid"
|
||||||
|
// ConcurrencyReplace kills the currently running SparkApplication instance and replaces it with a new one.
|
||||||
|
ConcurrencyReplace ConcurrencyPolicy = "Replace"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ScheduledSparkApplicationSpec struct {
|
||||||
|
// Schedule is a cron schedule on which the application should run.
|
||||||
|
Schedule string `json:"schedule"`
|
||||||
|
// Template is a template from which SparkApplication instances can be created.
|
||||||
|
Template SparkApplicationSpec `json:"template"`
|
||||||
|
// Suspend is a flag telling the controller to suspend subsequent runs of the application if set to true.
|
||||||
|
Suspend *bool `json:"suspend,omitempty"`
|
||||||
|
// ConcurrencyPolicy is the policy governing concurrent SparkApplication runs.
|
||||||
|
ConcurrencyPolicy ConcurrencyPolicy `json:"concurrencyPolicy,omitempty"`
|
||||||
|
// RunHistoryLimit specifies the number of past runs of the application to remember.
|
||||||
|
RunHistoryLimit *int32 `json:"runHistoryLimit,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ScheduleState string
|
||||||
|
|
||||||
|
const (
|
||||||
|
FailedValidationState ScheduleState = "FailedValidation"
|
||||||
|
ScheduledState ScheduleState = "Scheduled"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ScheduledSparkApplicationStatus struct {
|
||||||
|
// LastRun is the time when the last run of the application started.
|
||||||
|
LastRun metav1.Time `json:"lastRun,omitempty"`
|
||||||
|
// NextRun is the time when the next run of the application will start.
|
||||||
|
NextRun metav1.Time `json:"nextRun,omitempty"`
|
||||||
|
// PastRunNames keeps the names of SparkApplications for past runs.
|
||||||
|
// It keeps up to Spec.RunHistoryLimit number of past SparkApplication names,
|
||||||
|
// in reverse order of time when the SparkApplications get created.
|
||||||
|
PastRunNames []string `json:"pastRunNames,omitempty"`
|
||||||
|
// ScheduleState is the current scheduling state of the application.
|
||||||
|
ScheduleState ScheduleState `json:"scheduleState,omitempty"`
|
||||||
|
// Reason tells why the ScheduledSparkApplication is in the particular ScheduleState.
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationList carries a list of ScheduledSparkApplication objects.
|
||||||
|
type ScheduledSparkApplicationList struct {
|
||||||
|
metav1.TypeMeta `json:",inline"`
|
||||||
|
metav1.ListMeta `json:"metadata,omitempty"`
|
||||||
|
Items []ScheduledSparkApplication `json:"items,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// +genclient
|
||||||
|
// +genclient:noStatus
|
||||||
|
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||||
|
// +k8s:defaulter-gen=true
|
||||||
|
|
||||||
// SparkApplication represents a Spark application running on and using Kubernetes as a cluster manager.
|
// SparkApplication represents a Spark application running on and using Kubernetes as a cluster manager.
|
||||||
type SparkApplication struct {
|
type SparkApplication struct {
|
||||||
metav1.TypeMeta `json:",inline"`
|
metav1.TypeMeta `json:",inline"`
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
// +build !ignore_autogenerated
|
// +build !ignore_autogenerated
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was autogenerated by deepcopy-gen. Do not edit it manually!
|
// Code generated by deepcopy-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
|
|
@ -201,6 +201,125 @@ func (in *NamePath) DeepCopy() *NamePath {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *ScheduledSparkApplication) DeepCopyInto(out *ScheduledSparkApplication) {
|
||||||
|
*out = *in
|
||||||
|
out.TypeMeta = in.TypeMeta
|
||||||
|
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
|
||||||
|
in.Spec.DeepCopyInto(&out.Spec)
|
||||||
|
in.Status.DeepCopyInto(&out.Status)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplication.
|
||||||
|
func (in *ScheduledSparkApplication) DeepCopy() *ScheduledSparkApplication {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(ScheduledSparkApplication)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||||
|
func (in *ScheduledSparkApplication) DeepCopyObject() runtime.Object {
|
||||||
|
if c := in.DeepCopy(); c != nil {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *ScheduledSparkApplicationList) DeepCopyInto(out *ScheduledSparkApplicationList) {
|
||||||
|
*out = *in
|
||||||
|
out.TypeMeta = in.TypeMeta
|
||||||
|
out.ListMeta = in.ListMeta
|
||||||
|
if in.Items != nil {
|
||||||
|
in, out := &in.Items, &out.Items
|
||||||
|
*out = make([]ScheduledSparkApplication, len(*in))
|
||||||
|
for i := range *in {
|
||||||
|
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationList.
|
||||||
|
func (in *ScheduledSparkApplicationList) DeepCopy() *ScheduledSparkApplicationList {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(ScheduledSparkApplicationList)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||||
|
func (in *ScheduledSparkApplicationList) DeepCopyObject() runtime.Object {
|
||||||
|
if c := in.DeepCopy(); c != nil {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *ScheduledSparkApplicationSpec) DeepCopyInto(out *ScheduledSparkApplicationSpec) {
|
||||||
|
*out = *in
|
||||||
|
in.Template.DeepCopyInto(&out.Template)
|
||||||
|
if in.Suspend != nil {
|
||||||
|
in, out := &in.Suspend, &out.Suspend
|
||||||
|
if *in == nil {
|
||||||
|
*out = nil
|
||||||
|
} else {
|
||||||
|
*out = new(bool)
|
||||||
|
**out = **in
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if in.RunHistoryLimit != nil {
|
||||||
|
in, out := &in.RunHistoryLimit, &out.RunHistoryLimit
|
||||||
|
if *in == nil {
|
||||||
|
*out = nil
|
||||||
|
} else {
|
||||||
|
*out = new(int32)
|
||||||
|
**out = **in
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationSpec.
|
||||||
|
func (in *ScheduledSparkApplicationSpec) DeepCopy() *ScheduledSparkApplicationSpec {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(ScheduledSparkApplicationSpec)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *ScheduledSparkApplicationStatus) DeepCopyInto(out *ScheduledSparkApplicationStatus) {
|
||||||
|
*out = *in
|
||||||
|
in.LastRun.DeepCopyInto(&out.LastRun)
|
||||||
|
in.NextRun.DeepCopyInto(&out.NextRun)
|
||||||
|
if in.PastRunNames != nil {
|
||||||
|
in, out := &in.PastRunNames, &out.PastRunNames
|
||||||
|
*out = make([]string, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationStatus.
|
||||||
|
func (in *ScheduledSparkApplicationStatus) DeepCopy() *ScheduledSparkApplicationStatus {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(ScheduledSparkApplicationStatus)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
func (in *SecretInfo) DeepCopyInto(out *SecretInfo) {
|
func (in *SecretInfo) DeepCopyInto(out *SecretInfo) {
|
||||||
*out = *in
|
*out = *in
|
||||||
|
|
@ -241,9 +360,8 @@ func (in *SparkApplication) DeepCopy() *SparkApplication {
|
||||||
func (in *SparkApplication) DeepCopyObject() runtime.Object {
|
func (in *SparkApplication) DeepCopyObject() runtime.Object {
|
||||||
if c := in.DeepCopy(); c != nil {
|
if c := in.DeepCopy(); c != nil {
|
||||||
return c
|
return c
|
||||||
} else {
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
|
@ -275,9 +393,8 @@ func (in *SparkApplicationList) DeepCopy() *SparkApplicationList {
|
||||||
func (in *SparkApplicationList) DeepCopyObject() runtime.Object {
|
func (in *SparkApplicationList) DeepCopyObject() runtime.Object {
|
||||||
if c := in.DeepCopy(); c != nil {
|
if c := in.DeepCopy(); c != nil {
|
||||||
return c
|
return c
|
||||||
} else {
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,10 +14,11 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package versioned
|
package versioned
|
||||||
|
|
||||||
import (
|
import (
|
||||||
glog "github.com/golang/glog"
|
|
||||||
discovery "k8s.io/client-go/discovery"
|
discovery "k8s.io/client-go/discovery"
|
||||||
rest "k8s.io/client-go/rest"
|
rest "k8s.io/client-go/rest"
|
||||||
flowcontrol "k8s.io/client-go/util/flowcontrol"
|
flowcontrol "k8s.io/client-go/util/flowcontrol"
|
||||||
|
|
@ -72,7 +73,6 @@ func NewForConfig(c *rest.Config) (*Clientset, error) {
|
||||||
|
|
||||||
cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
|
cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to create the DiscoveryClient: %v", err)
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &cs, nil
|
return &cs, nil
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,5 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
// This package has the automatically generated clientset.
|
// This package has the automatically generated clientset.
|
||||||
package versioned
|
package versioned
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package fake
|
package fake
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,5 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
// This package has the automatically generated fake clientset.
|
// This package has the automatically generated fake clientset.
|
||||||
package fake
|
package fake
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package fake
|
package fake
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -38,7 +40,7 @@ func init() {
|
||||||
//
|
//
|
||||||
// import (
|
// import (
|
||||||
// "k8s.io/client-go/kubernetes"
|
// "k8s.io/client-go/kubernetes"
|
||||||
// clientsetscheme "k8s.io/client-go/kuberentes/scheme"
|
// clientsetscheme "k8s.io/client-go/kubernetes/scheme"
|
||||||
// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
|
// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
|
||||||
// )
|
// )
|
||||||
//
|
//
|
||||||
|
|
@ -49,5 +51,4 @@ func init() {
|
||||||
// correctly.
|
// correctly.
|
||||||
func AddToScheme(scheme *runtime.Scheme) {
|
func AddToScheme(scheme *runtime.Scheme) {
|
||||||
sparkoperatorv1alpha1.AddToScheme(scheme)
|
sparkoperatorv1alpha1.AddToScheme(scheme)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,5 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
// This package contains the scheme of the automatically generated clientset.
|
// This package contains the scheme of the automatically generated clientset.
|
||||||
package scheme
|
package scheme
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package scheme
|
package scheme
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -38,7 +40,7 @@ func init() {
|
||||||
//
|
//
|
||||||
// import (
|
// import (
|
||||||
// "k8s.io/client-go/kubernetes"
|
// "k8s.io/client-go/kubernetes"
|
||||||
// clientsetscheme "k8s.io/client-go/kuberentes/scheme"
|
// clientsetscheme "k8s.io/client-go/kubernetes/scheme"
|
||||||
// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
|
// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
|
||||||
// )
|
// )
|
||||||
//
|
//
|
||||||
|
|
@ -49,5 +51,4 @@ func init() {
|
||||||
// correctly.
|
// correctly.
|
||||||
func AddToScheme(scheme *runtime.Scheme) {
|
func AddToScheme(scheme *runtime.Scheme) {
|
||||||
sparkoperatorv1alpha1.AddToScheme(scheme)
|
sparkoperatorv1alpha1.AddToScheme(scheme)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,5 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
// This package has the automatically generated typed clients.
|
// This package has the automatically generated typed clients.
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,5 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
// Package fake has the automatically generated clients.
|
// Package fake has the automatically generated clients.
|
||||||
package fake
|
package fake
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,128 @@
|
||||||
|
/*
|
||||||
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
|
package fake
|
||||||
|
|
||||||
|
import (
|
||||||
|
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
labels "k8s.io/apimachinery/pkg/labels"
|
||||||
|
schema "k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
types "k8s.io/apimachinery/pkg/types"
|
||||||
|
watch "k8s.io/apimachinery/pkg/watch"
|
||||||
|
testing "k8s.io/client-go/testing"
|
||||||
|
v1alpha1 "k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FakeScheduledSparkApplications implements ScheduledSparkApplicationInterface
|
||||||
|
type FakeScheduledSparkApplications struct {
|
||||||
|
Fake *FakeSparkoperatorV1alpha1
|
||||||
|
ns string
|
||||||
|
}
|
||||||
|
|
||||||
|
var scheduledsparkapplicationsResource = schema.GroupVersionResource{Group: "sparkoperator", Version: "v1alpha1", Resource: "scheduledsparkapplications"}
|
||||||
|
|
||||||
|
var scheduledsparkapplicationsKind = schema.GroupVersionKind{Group: "sparkoperator", Version: "v1alpha1", Kind: "ScheduledSparkApplication"}
|
||||||
|
|
||||||
|
// Get takes name of the scheduledSparkApplication, and returns the corresponding scheduledSparkApplication object, and an error if there is any.
|
||||||
|
func (c *FakeScheduledSparkApplications) Get(name string, options v1.GetOptions) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
obj, err := c.Fake.
|
||||||
|
Invokes(testing.NewGetAction(scheduledsparkapplicationsResource, c.ns, name), &v1alpha1.ScheduledSparkApplication{})
|
||||||
|
|
||||||
|
if obj == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return obj.(*v1alpha1.ScheduledSparkApplication), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// List takes label and field selectors, and returns the list of ScheduledSparkApplications that match those selectors.
|
||||||
|
func (c *FakeScheduledSparkApplications) List(opts v1.ListOptions) (result *v1alpha1.ScheduledSparkApplicationList, err error) {
|
||||||
|
obj, err := c.Fake.
|
||||||
|
Invokes(testing.NewListAction(scheduledsparkapplicationsResource, scheduledsparkapplicationsKind, c.ns, opts), &v1alpha1.ScheduledSparkApplicationList{})
|
||||||
|
|
||||||
|
if obj == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
label, _, _ := testing.ExtractFromListOptions(opts)
|
||||||
|
if label == nil {
|
||||||
|
label = labels.Everything()
|
||||||
|
}
|
||||||
|
list := &v1alpha1.ScheduledSparkApplicationList{}
|
||||||
|
for _, item := range obj.(*v1alpha1.ScheduledSparkApplicationList).Items {
|
||||||
|
if label.Matches(labels.Set(item.Labels)) {
|
||||||
|
list.Items = append(list.Items, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return list, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watch returns a watch.Interface that watches the requested scheduledSparkApplications.
|
||||||
|
func (c *FakeScheduledSparkApplications) Watch(opts v1.ListOptions) (watch.Interface, error) {
|
||||||
|
return c.Fake.
|
||||||
|
InvokesWatch(testing.NewWatchAction(scheduledsparkapplicationsResource, c.ns, opts))
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create takes the representation of a scheduledSparkApplication and creates it. Returns the server's representation of the scheduledSparkApplication, and an error, if there is any.
|
||||||
|
func (c *FakeScheduledSparkApplications) Create(scheduledSparkApplication *v1alpha1.ScheduledSparkApplication) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
obj, err := c.Fake.
|
||||||
|
Invokes(testing.NewCreateAction(scheduledsparkapplicationsResource, c.ns, scheduledSparkApplication), &v1alpha1.ScheduledSparkApplication{})
|
||||||
|
|
||||||
|
if obj == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return obj.(*v1alpha1.ScheduledSparkApplication), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update takes the representation of a scheduledSparkApplication and updates it. Returns the server's representation of the scheduledSparkApplication, and an error, if there is any.
|
||||||
|
func (c *FakeScheduledSparkApplications) Update(scheduledSparkApplication *v1alpha1.ScheduledSparkApplication) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
obj, err := c.Fake.
|
||||||
|
Invokes(testing.NewUpdateAction(scheduledsparkapplicationsResource, c.ns, scheduledSparkApplication), &v1alpha1.ScheduledSparkApplication{})
|
||||||
|
|
||||||
|
if obj == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return obj.(*v1alpha1.ScheduledSparkApplication), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete takes name of the scheduledSparkApplication and deletes it. Returns an error if one occurs.
|
||||||
|
func (c *FakeScheduledSparkApplications) Delete(name string, options *v1.DeleteOptions) error {
|
||||||
|
_, err := c.Fake.
|
||||||
|
Invokes(testing.NewDeleteAction(scheduledsparkapplicationsResource, c.ns, name), &v1alpha1.ScheduledSparkApplication{})
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteCollection deletes a collection of objects.
|
||||||
|
func (c *FakeScheduledSparkApplications) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
|
||||||
|
action := testing.NewDeleteCollectionAction(scheduledsparkapplicationsResource, c.ns, listOptions)
|
||||||
|
|
||||||
|
_, err := c.Fake.Invokes(action, &v1alpha1.ScheduledSparkApplicationList{})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Patch applies the patch and returns the patched scheduledSparkApplication.
|
||||||
|
func (c *FakeScheduledSparkApplications) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
obj, err := c.Fake.
|
||||||
|
Invokes(testing.NewPatchSubresourceAction(scheduledsparkapplicationsResource, c.ns, name, data, subresources...), &v1alpha1.ScheduledSparkApplication{})
|
||||||
|
|
||||||
|
if obj == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return obj.(*v1alpha1.ScheduledSparkApplication), err
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package fake
|
package fake
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package fake
|
package fake
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -26,6 +28,10 @@ type FakeSparkoperatorV1alpha1 struct {
|
||||||
*testing.Fake
|
*testing.Fake
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *FakeSparkoperatorV1alpha1) ScheduledSparkApplications(namespace string) v1alpha1.ScheduledSparkApplicationInterface {
|
||||||
|
return &FakeScheduledSparkApplications{c, namespace}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *FakeSparkoperatorV1alpha1) SparkApplications(namespace string) v1alpha1.SparkApplicationInterface {
|
func (c *FakeSparkoperatorV1alpha1) SparkApplications(namespace string) v1alpha1.SparkApplicationInterface {
|
||||||
return &FakeSparkApplications{c, namespace}
|
return &FakeSparkApplications{c, namespace}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,10 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
|
type ScheduledSparkApplicationExpansion interface{}
|
||||||
|
|
||||||
type SparkApplicationExpansion interface{}
|
type SparkApplicationExpansion interface{}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,157 @@
|
||||||
|
/*
|
||||||
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
|
package v1alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
types "k8s.io/apimachinery/pkg/types"
|
||||||
|
watch "k8s.io/apimachinery/pkg/watch"
|
||||||
|
rest "k8s.io/client-go/rest"
|
||||||
|
v1alpha1 "k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
scheme "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned/scheme"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationsGetter has a method to return a ScheduledSparkApplicationInterface.
|
||||||
|
// A group's client should implement this interface.
|
||||||
|
type ScheduledSparkApplicationsGetter interface {
|
||||||
|
ScheduledSparkApplications(namespace string) ScheduledSparkApplicationInterface
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationInterface has methods to work with ScheduledSparkApplication resources.
|
||||||
|
type ScheduledSparkApplicationInterface interface {
|
||||||
|
Create(*v1alpha1.ScheduledSparkApplication) (*v1alpha1.ScheduledSparkApplication, error)
|
||||||
|
Update(*v1alpha1.ScheduledSparkApplication) (*v1alpha1.ScheduledSparkApplication, error)
|
||||||
|
Delete(name string, options *v1.DeleteOptions) error
|
||||||
|
DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
|
||||||
|
Get(name string, options v1.GetOptions) (*v1alpha1.ScheduledSparkApplication, error)
|
||||||
|
List(opts v1.ListOptions) (*v1alpha1.ScheduledSparkApplicationList, error)
|
||||||
|
Watch(opts v1.ListOptions) (watch.Interface, error)
|
||||||
|
Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ScheduledSparkApplication, err error)
|
||||||
|
ScheduledSparkApplicationExpansion
|
||||||
|
}
|
||||||
|
|
||||||
|
// scheduledSparkApplications implements ScheduledSparkApplicationInterface
|
||||||
|
type scheduledSparkApplications struct {
|
||||||
|
client rest.Interface
|
||||||
|
ns string
|
||||||
|
}
|
||||||
|
|
||||||
|
// newScheduledSparkApplications returns a ScheduledSparkApplications
|
||||||
|
func newScheduledSparkApplications(c *SparkoperatorV1alpha1Client, namespace string) *scheduledSparkApplications {
|
||||||
|
return &scheduledSparkApplications{
|
||||||
|
client: c.RESTClient(),
|
||||||
|
ns: namespace,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get takes name of the scheduledSparkApplication, and returns the corresponding scheduledSparkApplication object, and an error if there is any.
|
||||||
|
func (c *scheduledSparkApplications) Get(name string, options v1.GetOptions) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
result = &v1alpha1.ScheduledSparkApplication{}
|
||||||
|
err = c.client.Get().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
Name(name).
|
||||||
|
VersionedParams(&options, scheme.ParameterCodec).
|
||||||
|
Do().
|
||||||
|
Into(result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// List takes label and field selectors, and returns the list of ScheduledSparkApplications that match those selectors.
|
||||||
|
func (c *scheduledSparkApplications) List(opts v1.ListOptions) (result *v1alpha1.ScheduledSparkApplicationList, err error) {
|
||||||
|
result = &v1alpha1.ScheduledSparkApplicationList{}
|
||||||
|
err = c.client.Get().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
VersionedParams(&opts, scheme.ParameterCodec).
|
||||||
|
Do().
|
||||||
|
Into(result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watch returns a watch.Interface that watches the requested scheduledSparkApplications.
|
||||||
|
func (c *scheduledSparkApplications) Watch(opts v1.ListOptions) (watch.Interface, error) {
|
||||||
|
opts.Watch = true
|
||||||
|
return c.client.Get().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
VersionedParams(&opts, scheme.ParameterCodec).
|
||||||
|
Watch()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create takes the representation of a scheduledSparkApplication and creates it. Returns the server's representation of the scheduledSparkApplication, and an error, if there is any.
|
||||||
|
func (c *scheduledSparkApplications) Create(scheduledSparkApplication *v1alpha1.ScheduledSparkApplication) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
result = &v1alpha1.ScheduledSparkApplication{}
|
||||||
|
err = c.client.Post().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
Body(scheduledSparkApplication).
|
||||||
|
Do().
|
||||||
|
Into(result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update takes the representation of a scheduledSparkApplication and updates it. Returns the server's representation of the scheduledSparkApplication, and an error, if there is any.
|
||||||
|
func (c *scheduledSparkApplications) Update(scheduledSparkApplication *v1alpha1.ScheduledSparkApplication) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
result = &v1alpha1.ScheduledSparkApplication{}
|
||||||
|
err = c.client.Put().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
Name(scheduledSparkApplication.Name).
|
||||||
|
Body(scheduledSparkApplication).
|
||||||
|
Do().
|
||||||
|
Into(result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete takes name of the scheduledSparkApplication and deletes it. Returns an error if one occurs.
|
||||||
|
func (c *scheduledSparkApplications) Delete(name string, options *v1.DeleteOptions) error {
|
||||||
|
return c.client.Delete().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
Name(name).
|
||||||
|
Body(options).
|
||||||
|
Do().
|
||||||
|
Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteCollection deletes a collection of objects.
|
||||||
|
func (c *scheduledSparkApplications) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
|
||||||
|
return c.client.Delete().
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
VersionedParams(&listOptions, scheme.ParameterCodec).
|
||||||
|
Body(options).
|
||||||
|
Do().
|
||||||
|
Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Patch applies the patch and returns the patched scheduledSparkApplication.
|
||||||
|
func (c *scheduledSparkApplications) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
result = &v1alpha1.ScheduledSparkApplication{}
|
||||||
|
err = c.client.Patch(pt).
|
||||||
|
Namespace(c.ns).
|
||||||
|
Resource("scheduledsparkapplications").
|
||||||
|
SubResource(subresources...).
|
||||||
|
Name(name).
|
||||||
|
Body(data).
|
||||||
|
Do().
|
||||||
|
Into(result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// Code generated by client-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -25,6 +27,7 @@ import (
|
||||||
|
|
||||||
type SparkoperatorV1alpha1Interface interface {
|
type SparkoperatorV1alpha1Interface interface {
|
||||||
RESTClient() rest.Interface
|
RESTClient() rest.Interface
|
||||||
|
ScheduledSparkApplicationsGetter
|
||||||
SparkApplicationsGetter
|
SparkApplicationsGetter
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -33,6 +36,10 @@ type SparkoperatorV1alpha1Client struct {
|
||||||
restClient rest.Interface
|
restClient rest.Interface
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *SparkoperatorV1alpha1Client) ScheduledSparkApplications(namespace string) ScheduledSparkApplicationInterface {
|
||||||
|
return newScheduledSparkApplications(c, namespace)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *SparkoperatorV1alpha1Client) SparkApplications(namespace string) SparkApplicationInterface {
|
func (c *SparkoperatorV1alpha1Client) SparkApplications(namespace string) SparkApplicationInterface {
|
||||||
return newSparkApplications(c, namespace)
|
return newSparkApplications(c, namespace)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by informer-gen
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package externalversions
|
package externalversions
|
||||||
|
|
||||||
|
|
@ -32,12 +32,16 @@ import (
|
||||||
sparkoperator_k8s_io "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions/sparkoperator.k8s.io"
|
sparkoperator_k8s_io "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions/sparkoperator.k8s.io"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// SharedInformerOption defines the functional option type for SharedInformerFactory.
|
||||||
|
type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory
|
||||||
|
|
||||||
type sharedInformerFactory struct {
|
type sharedInformerFactory struct {
|
||||||
client versioned.Interface
|
client versioned.Interface
|
||||||
namespace string
|
namespace string
|
||||||
tweakListOptions internalinterfaces.TweakListOptionsFunc
|
tweakListOptions internalinterfaces.TweakListOptionsFunc
|
||||||
lock sync.Mutex
|
lock sync.Mutex
|
||||||
defaultResync time.Duration
|
defaultResync time.Duration
|
||||||
|
customResync map[reflect.Type]time.Duration
|
||||||
|
|
||||||
informers map[reflect.Type]cache.SharedIndexInformer
|
informers map[reflect.Type]cache.SharedIndexInformer
|
||||||
// startedInformers is used for tracking which informers have been started.
|
// startedInformers is used for tracking which informers have been started.
|
||||||
|
|
@ -45,23 +49,62 @@ type sharedInformerFactory struct {
|
||||||
startedInformers map[reflect.Type]bool
|
startedInformers map[reflect.Type]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSharedInformerFactory constructs a new instance of sharedInformerFactory
|
// WithCustomResyncConfig sets a custom resync period for the specified informer types.
|
||||||
|
func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption {
|
||||||
|
return func(factory *sharedInformerFactory) *sharedInformerFactory {
|
||||||
|
for k, v := range resyncConfig {
|
||||||
|
factory.customResync[reflect.TypeOf(k)] = v
|
||||||
|
}
|
||||||
|
return factory
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory.
|
||||||
|
func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption {
|
||||||
|
return func(factory *sharedInformerFactory) *sharedInformerFactory {
|
||||||
|
factory.tweakListOptions = tweakListOptions
|
||||||
|
return factory
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithNamespace limits the SharedInformerFactory to the specified namespace.
|
||||||
|
func WithNamespace(namespace string) SharedInformerOption {
|
||||||
|
return func(factory *sharedInformerFactory) *sharedInformerFactory {
|
||||||
|
factory.namespace = namespace
|
||||||
|
return factory
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces.
|
||||||
func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory {
|
func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory {
|
||||||
return NewFilteredSharedInformerFactory(client, defaultResync, v1.NamespaceAll, nil)
|
return NewSharedInformerFactoryWithOptions(client, defaultResync)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.
|
// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.
|
||||||
// Listers obtained via this SharedInformerFactory will be subject to the same filters
|
// Listers obtained via this SharedInformerFactory will be subject to the same filters
|
||||||
// as specified here.
|
// as specified here.
|
||||||
|
// Deprecated: Please use NewSharedInformerFactoryWithOptions instead
|
||||||
func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {
|
func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {
|
||||||
return &sharedInformerFactory{
|
return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options.
|
||||||
|
func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {
|
||||||
|
factory := &sharedInformerFactory{
|
||||||
client: client,
|
client: client,
|
||||||
namespace: namespace,
|
namespace: v1.NamespaceAll,
|
||||||
tweakListOptions: tweakListOptions,
|
|
||||||
defaultResync: defaultResync,
|
defaultResync: defaultResync,
|
||||||
informers: make(map[reflect.Type]cache.SharedIndexInformer),
|
informers: make(map[reflect.Type]cache.SharedIndexInformer),
|
||||||
startedInformers: make(map[reflect.Type]bool),
|
startedInformers: make(map[reflect.Type]bool),
|
||||||
|
customResync: make(map[reflect.Type]time.Duration),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Apply all options
|
||||||
|
for _, opt := range options {
|
||||||
|
factory = opt(factory)
|
||||||
|
}
|
||||||
|
|
||||||
|
return factory
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start initializes all requested informers.
|
// Start initializes all requested informers.
|
||||||
|
|
@ -110,7 +153,13 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal
|
||||||
if exists {
|
if exists {
|
||||||
return informer
|
return informer
|
||||||
}
|
}
|
||||||
informer = newFunc(f.client, f.defaultResync)
|
|
||||||
|
resyncPeriod, exists := f.customResync[informerType]
|
||||||
|
if !exists {
|
||||||
|
resyncPeriod = f.defaultResync
|
||||||
|
}
|
||||||
|
|
||||||
|
informer = newFunc(f.client, resyncPeriod)
|
||||||
f.informers[informerType] = informer
|
f.informers[informerType] = informer
|
||||||
|
|
||||||
return informer
|
return informer
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by informer-gen
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package externalversions
|
package externalversions
|
||||||
|
|
||||||
|
|
@ -53,6 +53,8 @@ func (f *genericInformer) Lister() cache.GenericLister {
|
||||||
func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
|
func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
|
||||||
switch resource {
|
switch resource {
|
||||||
// Group=sparkoperator, Version=v1alpha1
|
// Group=sparkoperator, Version=v1alpha1
|
||||||
|
case v1alpha1.SchemeGroupVersion.WithResource("scheduledsparkapplications"):
|
||||||
|
return &genericInformer{resource: resource.GroupResource(), informer: f.Sparkoperator().V1alpha1().ScheduledSparkApplications().Informer()}, nil
|
||||||
case v1alpha1.SchemeGroupVersion.WithResource("sparkapplications"):
|
case v1alpha1.SchemeGroupVersion.WithResource("sparkapplications"):
|
||||||
return &genericInformer{resource: resource.GroupResource(), informer: f.Sparkoperator().V1alpha1().SparkApplications().Informer()}, nil
|
return &genericInformer{resource: resource.GroupResource(), informer: f.Sparkoperator().V1alpha1().SparkApplications().Informer()}, nil
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by informer-gen
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package internalinterfaces
|
package internalinterfaces
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by informer-gen
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package sparkoperator
|
package sparkoperator
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by informer-gen
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
|
|
@ -24,6 +24,8 @@ import (
|
||||||
|
|
||||||
// Interface provides access to all the informers in this group version.
|
// Interface provides access to all the informers in this group version.
|
||||||
type Interface interface {
|
type Interface interface {
|
||||||
|
// ScheduledSparkApplications returns a ScheduledSparkApplicationInformer.
|
||||||
|
ScheduledSparkApplications() ScheduledSparkApplicationInformer
|
||||||
// SparkApplications returns a SparkApplicationInformer.
|
// SparkApplications returns a SparkApplicationInformer.
|
||||||
SparkApplications() SparkApplicationInformer
|
SparkApplications() SparkApplicationInformer
|
||||||
}
|
}
|
||||||
|
|
@ -39,6 +41,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList
|
||||||
return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
|
return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ScheduledSparkApplications returns a ScheduledSparkApplicationInformer.
|
||||||
|
func (v *version) ScheduledSparkApplications() ScheduledSparkApplicationInformer {
|
||||||
|
return &scheduledSparkApplicationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
|
||||||
|
}
|
||||||
|
|
||||||
// SparkApplications returns a SparkApplicationInformer.
|
// SparkApplications returns a SparkApplicationInformer.
|
||||||
func (v *version) SparkApplications() SparkApplicationInformer {
|
func (v *version) SparkApplications() SparkApplicationInformer {
|
||||||
return &sparkApplicationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
|
return &sparkApplicationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,89 @@
|
||||||
|
/*
|
||||||
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
|
package v1alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
time "time"
|
||||||
|
|
||||||
|
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
runtime "k8s.io/apimachinery/pkg/runtime"
|
||||||
|
watch "k8s.io/apimachinery/pkg/watch"
|
||||||
|
cache "k8s.io/client-go/tools/cache"
|
||||||
|
sparkoperator_k8s_io_v1alpha1 "k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
versioned "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned"
|
||||||
|
internalinterfaces "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions/internalinterfaces"
|
||||||
|
v1alpha1 "k8s.io/spark-on-k8s-operator/pkg/client/listers/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationInformer provides access to a shared informer and lister for
|
||||||
|
// ScheduledSparkApplications.
|
||||||
|
type ScheduledSparkApplicationInformer interface {
|
||||||
|
Informer() cache.SharedIndexInformer
|
||||||
|
Lister() v1alpha1.ScheduledSparkApplicationLister
|
||||||
|
}
|
||||||
|
|
||||||
|
type scheduledSparkApplicationInformer struct {
|
||||||
|
factory internalinterfaces.SharedInformerFactory
|
||||||
|
tweakListOptions internalinterfaces.TweakListOptionsFunc
|
||||||
|
namespace string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewScheduledSparkApplicationInformer constructs a new informer for ScheduledSparkApplication type.
|
||||||
|
// Always prefer using an informer factory to get a shared informer instead of getting an independent
|
||||||
|
// one. This reduces memory footprint and number of connections to the server.
|
||||||
|
func NewScheduledSparkApplicationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
|
||||||
|
return NewFilteredScheduledSparkApplicationInformer(client, namespace, resyncPeriod, indexers, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFilteredScheduledSparkApplicationInformer constructs a new informer for ScheduledSparkApplication type.
|
||||||
|
// Always prefer using an informer factory to get a shared informer instead of getting an independent
|
||||||
|
// one. This reduces memory footprint and number of connections to the server.
|
||||||
|
func NewFilteredScheduledSparkApplicationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
|
||||||
|
return cache.NewSharedIndexInformer(
|
||||||
|
&cache.ListWatch{
|
||||||
|
ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
|
||||||
|
if tweakListOptions != nil {
|
||||||
|
tweakListOptions(&options)
|
||||||
|
}
|
||||||
|
return client.SparkoperatorV1alpha1().ScheduledSparkApplications(namespace).List(options)
|
||||||
|
},
|
||||||
|
WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
|
||||||
|
if tweakListOptions != nil {
|
||||||
|
tweakListOptions(&options)
|
||||||
|
}
|
||||||
|
return client.SparkoperatorV1alpha1().ScheduledSparkApplications(namespace).Watch(options)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
&sparkoperator_k8s_io_v1alpha1.ScheduledSparkApplication{},
|
||||||
|
resyncPeriod,
|
||||||
|
indexers,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *scheduledSparkApplicationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
|
||||||
|
return NewFilteredScheduledSparkApplicationInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *scheduledSparkApplicationInformer) Informer() cache.SharedIndexInformer {
|
||||||
|
return f.factory.InformerFor(&sparkoperator_k8s_io_v1alpha1.ScheduledSparkApplication{}, f.defaultInformer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *scheduledSparkApplicationInformer) Lister() v1alpha1.ScheduledSparkApplicationLister {
|
||||||
|
return v1alpha1.NewScheduledSparkApplicationLister(f.Informer().GetIndexer())
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by informer-gen
|
// Code generated by informer-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,10 +14,18 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by lister-gen
|
// Code generated by lister-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationListerExpansion allows custom methods to be added to
|
||||||
|
// ScheduledSparkApplicationLister.
|
||||||
|
type ScheduledSparkApplicationListerExpansion interface{}
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationNamespaceListerExpansion allows custom methods to be added to
|
||||||
|
// ScheduledSparkApplicationNamespaceLister.
|
||||||
|
type ScheduledSparkApplicationNamespaceListerExpansion interface{}
|
||||||
|
|
||||||
// SparkApplicationListerExpansion allows custom methods to be added to
|
// SparkApplicationListerExpansion allows custom methods to be added to
|
||||||
// SparkApplicationLister.
|
// SparkApplicationLister.
|
||||||
type SparkApplicationListerExpansion interface{}
|
type SparkApplicationListerExpansion interface{}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
/*
|
||||||
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Code generated by lister-gen. DO NOT EDIT.
|
||||||
|
|
||||||
|
package v1alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
|
"k8s.io/client-go/tools/cache"
|
||||||
|
v1alpha1 "k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationLister helps list ScheduledSparkApplications.
|
||||||
|
type ScheduledSparkApplicationLister interface {
|
||||||
|
// List lists all ScheduledSparkApplications in the indexer.
|
||||||
|
List(selector labels.Selector) (ret []*v1alpha1.ScheduledSparkApplication, err error)
|
||||||
|
// ScheduledSparkApplications returns an object that can list and get ScheduledSparkApplications.
|
||||||
|
ScheduledSparkApplications(namespace string) ScheduledSparkApplicationNamespaceLister
|
||||||
|
ScheduledSparkApplicationListerExpansion
|
||||||
|
}
|
||||||
|
|
||||||
|
// scheduledSparkApplicationLister implements the ScheduledSparkApplicationLister interface.
|
||||||
|
type scheduledSparkApplicationLister struct {
|
||||||
|
indexer cache.Indexer
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewScheduledSparkApplicationLister returns a new ScheduledSparkApplicationLister.
|
||||||
|
func NewScheduledSparkApplicationLister(indexer cache.Indexer) ScheduledSparkApplicationLister {
|
||||||
|
return &scheduledSparkApplicationLister{indexer: indexer}
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists all ScheduledSparkApplications in the indexer.
|
||||||
|
func (s *scheduledSparkApplicationLister) List(selector labels.Selector) (ret []*v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
err = cache.ListAll(s.indexer, selector, func(m interface{}) {
|
||||||
|
ret = append(ret, m.(*v1alpha1.ScheduledSparkApplication))
|
||||||
|
})
|
||||||
|
return ret, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduledSparkApplications returns an object that can list and get ScheduledSparkApplications.
|
||||||
|
func (s *scheduledSparkApplicationLister) ScheduledSparkApplications(namespace string) ScheduledSparkApplicationNamespaceLister {
|
||||||
|
return scheduledSparkApplicationNamespaceLister{indexer: s.indexer, namespace: namespace}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScheduledSparkApplicationNamespaceLister helps list and get ScheduledSparkApplications.
|
||||||
|
type ScheduledSparkApplicationNamespaceLister interface {
|
||||||
|
// List lists all ScheduledSparkApplications in the indexer for a given namespace.
|
||||||
|
List(selector labels.Selector) (ret []*v1alpha1.ScheduledSparkApplication, err error)
|
||||||
|
// Get retrieves the ScheduledSparkApplication from the indexer for a given namespace and name.
|
||||||
|
Get(name string) (*v1alpha1.ScheduledSparkApplication, error)
|
||||||
|
ScheduledSparkApplicationNamespaceListerExpansion
|
||||||
|
}
|
||||||
|
|
||||||
|
// scheduledSparkApplicationNamespaceLister implements the ScheduledSparkApplicationNamespaceLister
|
||||||
|
// interface.
|
||||||
|
type scheduledSparkApplicationNamespaceLister struct {
|
||||||
|
indexer cache.Indexer
|
||||||
|
namespace string
|
||||||
|
}
|
||||||
|
|
||||||
|
// List lists all ScheduledSparkApplications in the indexer for a given namespace.
|
||||||
|
func (s scheduledSparkApplicationNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.ScheduledSparkApplication, err error) {
|
||||||
|
err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
|
||||||
|
ret = append(ret, m.(*v1alpha1.ScheduledSparkApplication))
|
||||||
|
})
|
||||||
|
return ret, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get retrieves the ScheduledSparkApplication from the indexer for a given namespace and name.
|
||||||
|
func (s scheduledSparkApplicationNamespaceLister) Get(name string) (*v1alpha1.ScheduledSparkApplication, error) {
|
||||||
|
obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return nil, errors.NewNotFound(v1alpha1.Resource("scheduledsparkapplication"), name)
|
||||||
|
}
|
||||||
|
return obj.(*v1alpha1.ScheduledSparkApplication), nil
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/*
|
/*
|
||||||
Copyright 2018 The Kubernetes Authors.
|
Copyright The Kubernetes Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// This file was automatically generated by lister-gen
|
// Code generated by lister-gen. DO NOT EDIT.
|
||||||
|
|
||||||
package v1alpha1
|
package v1alpha1
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -16,12 +16,20 @@ limitations under the License.
|
||||||
|
|
||||||
package controller
|
package controller
|
||||||
|
|
||||||
// Package controller implements the CustomResourceDefinition (CRD) controller for SparkApplications.
|
// Package controller implements the CustomResourceDefinition (CRD) controller for SparkApplications and
|
||||||
// The controller is responsible for watching SparkApplication objects and submitting Spark applications
|
// ScheduledSparkApplications.
|
||||||
// described by the specs in the objects on behalf of users. After an application is submitted, the
|
//
|
||||||
// controller monitors the application state and updates the status field of the SparkApplication object
|
// The ScheduledSparkApplication controller is responsible for watching ScheduledSparkApplications objects
|
||||||
// accordingly. The controller uses a sparkSubmitRunner to submit applications to run in the Kubernetes
|
// and scheduling them according to the cron schedule in the ScheduledSparkApplication specification. For
|
||||||
// cluster where Spark Operator runs. The sparkSubmitRunner maintains a set of workers, each of which is
|
// each ScheduledSparkApplication, the controller creates a new SparkApplication instance when the next run
|
||||||
// a goroutine, for actually running the spark-submit commands. The controller also uses a sparkPodMonitor
|
// of the application is due and the condition for starting the next run is satisfied.
|
||||||
// to watch Spark driver and executor pods. The sparkPodMonitor sends driver and executor state updates
|
//
|
||||||
// to the controller, which then updates status field of SparkApplication objects accordingly.
|
// The SparkApplication controller is responsible for watching SparkApplication objects and submitting
|
||||||
|
// Spark applications described by the specs in the objects on behalf of users. After an application is
|
||||||
|
// submitted, the controller monitors the application state and updates the status field of the
|
||||||
|
// SparkApplication object accordingly. The controller uses a sparkSubmitRunner to submit applications
|
||||||
|
// to run in the Kubernetes cluster where Spark Operator runs. The sparkSubmitRunner maintains a set of
|
||||||
|
// workers, each of which is a goroutine, for actually running the spark-submit commands. The controller
|
||||||
|
// also uses a sparkPodMonitor to watch Spark driver and executor pods. The sparkPodMonitor sends driver
|
||||||
|
// and executor state updates to the controller, which then updates status field of SparkApplication
|
||||||
|
// objects accordingly.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,381 @@
|
||||||
|
/*
|
||||||
|
Copyright 2018 Google LLC
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package scheduledsparkapplication
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/golang/glog"
|
||||||
|
"github.com/robfig/cron"
|
||||||
|
|
||||||
|
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||||
|
"k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/clock"
|
||||||
|
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||||
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
|
"k8s.io/client-go/kubernetes"
|
||||||
|
"k8s.io/client-go/kubernetes/scheme"
|
||||||
|
"k8s.io/client-go/tools/cache"
|
||||||
|
"k8s.io/client-go/util/retry"
|
||||||
|
"k8s.io/client-go/util/workqueue"
|
||||||
|
|
||||||
|
"k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
crdclientset "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned"
|
||||||
|
crdscheme "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned/scheme"
|
||||||
|
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
||||||
|
crdlisters "k8s.io/spark-on-k8s-operator/pkg/client/listers/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
|
||||||
|
)
|
||||||
|
|
||||||
|
type Controller struct {
|
||||||
|
crdClient crdclientset.Interface
|
||||||
|
kubeClient kubernetes.Interface
|
||||||
|
extensionsClient apiextensionsclient.Interface
|
||||||
|
queue workqueue.RateLimitingInterface
|
||||||
|
cacheSynced cache.InformerSynced
|
||||||
|
lister crdlisters.ScheduledSparkApplicationLister
|
||||||
|
clock clock.Clock
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewController(
|
||||||
|
crdClient crdclientset.Interface,
|
||||||
|
kubeClient kubernetes.Interface,
|
||||||
|
extensionsClient apiextensionsclient.Interface,
|
||||||
|
informerFactory crdinformers.SharedInformerFactory,
|
||||||
|
clock clock.Clock) *Controller {
|
||||||
|
crdscheme.AddToScheme(scheme.Scheme)
|
||||||
|
|
||||||
|
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(),
|
||||||
|
"scheduled-spark-application-controller")
|
||||||
|
|
||||||
|
controller := &Controller{
|
||||||
|
crdClient: crdClient,
|
||||||
|
kubeClient: kubeClient,
|
||||||
|
extensionsClient: extensionsClient,
|
||||||
|
queue: queue,
|
||||||
|
clock: clock,
|
||||||
|
}
|
||||||
|
|
||||||
|
informer := informerFactory.Sparkoperator().V1alpha1().ScheduledSparkApplications()
|
||||||
|
informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||||
|
AddFunc: controller.onAdd,
|
||||||
|
UpdateFunc: controller.onUpdate,
|
||||||
|
DeleteFunc: controller.onDelete,
|
||||||
|
})
|
||||||
|
controller.cacheSynced = informer.Informer().HasSynced
|
||||||
|
controller.lister = informer.Lister()
|
||||||
|
|
||||||
|
return controller
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) Start(workers int, stopCh <-chan struct{}) error {
|
||||||
|
glog.Info("Starting the ScheduledSparkApplication controller")
|
||||||
|
|
||||||
|
if !cache.WaitForCacheSync(stopCh, c.cacheSynced) {
|
||||||
|
return fmt.Errorf("timed out waiting for cache to sync")
|
||||||
|
}
|
||||||
|
|
||||||
|
glog.Info("Starting the workers of the ScheduledSparkApplication controller")
|
||||||
|
for i := 0; i < workers; i++ {
|
||||||
|
// runWorker will loop until "something bad" happens. Until will then rekick
|
||||||
|
// the worker after one second.
|
||||||
|
go wait.Until(c.runWorker, time.Second, stopCh)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) Stop() {
|
||||||
|
glog.Info("Stopping the ScheduledSparkApplication controller")
|
||||||
|
c.queue.ShutDown()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) runWorker() {
|
||||||
|
defer utilruntime.HandleCrash()
|
||||||
|
for c.processNextItem() {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) processNextItem() bool {
|
||||||
|
key, quit := c.queue.Get()
|
||||||
|
if quit {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer c.queue.Done(key)
|
||||||
|
|
||||||
|
err := c.syncScheduledSparkApplication(key.(string))
|
||||||
|
if err == nil {
|
||||||
|
// Successfully processed the key or the key was not found so tell the queue to stop tracking
|
||||||
|
// history for your key. This will reset things like failure counts for per-item rate limiting.
|
||||||
|
c.queue.Forget(key)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// There was a failure so be sure to report it. This method allows for pluggable error handling
|
||||||
|
// which can be used for things like cluster-monitoring
|
||||||
|
utilruntime.HandleError(fmt.Errorf("failed to sync ScheduledSparkApplication %q: %v", key, err))
|
||||||
|
// Since we failed, we should requeue the item to work on later. This method will add a backoff
|
||||||
|
// to avoid hot-looping on particular items (they're probably still not going to work right away)
|
||||||
|
// and overall controller protection (everything I've done is broken, this controller needs to
|
||||||
|
// calm down or it can starve other useful work) cases.
|
||||||
|
c.queue.AddRateLimited(key)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) syncScheduledSparkApplication(key string) error {
|
||||||
|
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
app, err := c.lister.ScheduledSparkApplications(namespace).Get(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if app.Spec.Suspend != nil && *app.Spec.Suspend {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
glog.V(2).Infof("Syncing ScheduledSparkApplication %s", app.Name)
|
||||||
|
status := app.Status.DeepCopy()
|
||||||
|
schedule, err := cron.ParseStandard(app.Spec.Schedule)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to parse schedule %s of %s: %v", app.Spec.Schedule, app.Name, err)
|
||||||
|
status.ScheduleState = v1alpha1.FailedValidationState
|
||||||
|
status.Reason = err.Error()
|
||||||
|
} else {
|
||||||
|
status.ScheduleState = v1alpha1.ScheduledState
|
||||||
|
nextRunTime := status.NextRun.Time
|
||||||
|
if nextRunTime.IsZero() {
|
||||||
|
nextRunTime = schedule.Next(status.LastRun.Time)
|
||||||
|
status.NextRun = metav1.NewTime(nextRunTime)
|
||||||
|
}
|
||||||
|
now := c.clock.Now()
|
||||||
|
if nextRunTime.Before(now) {
|
||||||
|
// The next run is due. Check if this is the first run of the application.
|
||||||
|
if len(status.PastRunNames) == 0 {
|
||||||
|
// This is the first run of the application.
|
||||||
|
if err = c.startNextRun(app, status, schedule); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Check if the condition for starting the next run is satisfied.
|
||||||
|
ok, err := c.shouldStartNextRun(app)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
if err = c.startNextRun(app, status, schedule); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.updateScheduledSparkApplicationStatus(app, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) onAdd(obj interface{}) {
|
||||||
|
c.enqueue(obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) onUpdate(oldObj, newObj interface{}) {
|
||||||
|
c.enqueue(newObj)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) onDelete(obj interface{}) {
|
||||||
|
c.dequeue(obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) enqueue(obj interface{}) {
|
||||||
|
key, err := keyFunc(obj)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to get key for %v: %v", obj, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.queue.AddRateLimited(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) dequeue(obj interface{}) {
|
||||||
|
key, err := keyFunc(obj)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to get key for %v: %v", obj, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.queue.Forget(key)
|
||||||
|
c.queue.Done(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) createSparkApplication(
|
||||||
|
scheduledApp *v1alpha1.ScheduledSparkApplication, t time.Time) (string, error) {
|
||||||
|
app := &v1alpha1.SparkApplication{}
|
||||||
|
app.Spec = scheduledApp.Spec.Template
|
||||||
|
app.Name = fmt.Sprintf("%s-%d", scheduledApp.Name, t.UnixNano())
|
||||||
|
app.OwnerReferences = append(app.OwnerReferences, metav1.OwnerReference{
|
||||||
|
APIVersion: v1alpha1.SchemeGroupVersion.String(),
|
||||||
|
Kind: reflect.TypeOf(v1alpha1.ScheduledSparkApplication{}).Name(),
|
||||||
|
Name: scheduledApp.Name,
|
||||||
|
UID: scheduledApp.UID,
|
||||||
|
})
|
||||||
|
_, err := c.crdClient.SparkoperatorV1alpha1().SparkApplications(scheduledApp.Namespace).Create(app)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return app.Name, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) shouldStartNextRun(app *v1alpha1.ScheduledSparkApplication) (bool, error) {
|
||||||
|
switch app.Spec.ConcurrencyPolicy {
|
||||||
|
case v1alpha1.ConcurrencyAllow:
|
||||||
|
return true, nil
|
||||||
|
case v1alpha1.ConcurrencyForbid:
|
||||||
|
finished, _, err := c.hasLastRunFinished(app.Namespace, app.Status.PastRunNames[0])
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return finished, nil
|
||||||
|
case v1alpha1.ConcurrencyReplace:
|
||||||
|
if err := c.killLastRunIfNotFinished(app.Namespace, app.Status.PastRunNames[0]); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) startNextRun(
|
||||||
|
app *v1alpha1.ScheduledSparkApplication,
|
||||||
|
status *v1alpha1.ScheduledSparkApplicationStatus,
|
||||||
|
schedule cron.Schedule) error {
|
||||||
|
glog.Infof("Next run of %s is due, creating a new SparkApplication instance", app.Name)
|
||||||
|
status.LastRun = metav1.Now()
|
||||||
|
name, err := c.createSparkApplication(app, status.LastRun.Time)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to create a SparkApplication instance for %s: %v", app.Name, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
status.NextRun = metav1.NewTime(schedule.Next(status.LastRun.Time))
|
||||||
|
|
||||||
|
var limit int32 = 1
|
||||||
|
if app.Spec.RunHistoryLimit != nil {
|
||||||
|
limit = *app.Spec.RunHistoryLimit
|
||||||
|
}
|
||||||
|
|
||||||
|
rest := status.PastRunNames
|
||||||
|
var toDelete []string
|
||||||
|
if int32(len(status.PastRunNames)) >= limit {
|
||||||
|
rest = status.PastRunNames[:limit-1]
|
||||||
|
toDelete = status.PastRunNames[limit-1:]
|
||||||
|
}
|
||||||
|
// Pre-append the name of the latest run.
|
||||||
|
status.PastRunNames = append([]string{name}, rest...)
|
||||||
|
|
||||||
|
namespace := app.Namespace
|
||||||
|
// Delete runs that should no longer be kept.
|
||||||
|
for _, name := range toDelete {
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().SparkApplications(namespace).Delete(name, metav1.NewDeleteOptions(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) hasLastRunFinished(
|
||||||
|
namespace string,
|
||||||
|
lastRunName string) (bool, *v1alpha1.SparkApplication, error) {
|
||||||
|
app, err := c.crdClient.SparkoperatorV1alpha1().SparkApplications(namespace).Get(lastRunName, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
if errors.IsNotFound(err) {
|
||||||
|
return true, nil, nil
|
||||||
|
}
|
||||||
|
return false, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.Status.AppState.State == v1alpha1.CompletedState ||
|
||||||
|
app.Status.AppState.State == v1alpha1.FailedState, app, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) killLastRunIfNotFinished(namespace string, lastRunName string) error {
|
||||||
|
finished, app, err := c.hasLastRunFinished(namespace, lastRunName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if app == nil || finished {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete the driver pod of the last run if applicable.
|
||||||
|
if app.Status.DriverInfo.PodName != "" {
|
||||||
|
if err = c.kubeClient.CoreV1().Pods(namespace).Delete(app.Status.DriverInfo.PodName,
|
||||||
|
metav1.NewDeleteOptions(0)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete the SparkApplication object of the last run.
|
||||||
|
if err = c.crdClient.SparkoperatorV1alpha1().SparkApplications(namespace).Delete(lastRunName,
|
||||||
|
metav1.NewDeleteOptions(0)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) updateScheduledSparkApplicationStatus(
|
||||||
|
app *v1alpha1.ScheduledSparkApplication,
|
||||||
|
newStatus *v1alpha1.ScheduledSparkApplicationStatus) error {
|
||||||
|
// If the status has not changed, do not perform an update.
|
||||||
|
if isStatusEqual(newStatus, &app.Status) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
toUpdate := app.DeepCopy()
|
||||||
|
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||||
|
toUpdate.Status = *newStatus
|
||||||
|
_, updateErr := c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(toUpdate.Namespace).Update(
|
||||||
|
toUpdate)
|
||||||
|
if updateErr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(toUpdate.Namespace).Get(
|
||||||
|
toUpdate.Name, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
toUpdate = result
|
||||||
|
|
||||||
|
return updateErr
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func isStatusEqual(newStatus, currentStatus *v1alpha1.ScheduledSparkApplicationStatus) bool {
|
||||||
|
return newStatus.ScheduleState == currentStatus.ScheduleState &&
|
||||||
|
newStatus.LastRun == currentStatus.LastRun &&
|
||||||
|
newStatus.NextRun == currentStatus.NextRun &&
|
||||||
|
reflect.DeepEqual(newStatus.PastRunNames, currentStatus.PastRunNames) &&
|
||||||
|
newStatus.Reason == currentStatus.Reason
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,218 @@
|
||||||
|
/*
|
||||||
|
Copyright 2018 Google LLC
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package scheduledsparkapplication
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/robfig/cron"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
apiextensionsfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/clock"
|
||||||
|
kubeclientfake "k8s.io/client-go/kubernetes/fake"
|
||||||
|
"k8s.io/client-go/tools/cache"
|
||||||
|
|
||||||
|
"k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
crdclientfake "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned/fake"
|
||||||
|
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSyncScheduledSparkApplication(t *testing.T) {
|
||||||
|
var two int32 = 2
|
||||||
|
app := &v1alpha1.ScheduledSparkApplication{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Namespace: "default",
|
||||||
|
Name: "test-app",
|
||||||
|
},
|
||||||
|
Spec: v1alpha1.ScheduledSparkApplicationSpec{
|
||||||
|
Schedule: "@every 1m",
|
||||||
|
ConcurrencyPolicy: v1alpha1.ConcurrencyAllow,
|
||||||
|
RunHistoryLimit: &two,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c, informer, clk := newFakeController(app)
|
||||||
|
key, _ := cache.MetaNamespaceKeyFunc(app)
|
||||||
|
|
||||||
|
// This sync should start the first run.
|
||||||
|
err := c.syncScheduledSparkApplication(key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app, _ = c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(app.Namespace).Get(app.Name,
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.Equal(t, v1alpha1.ScheduledState, app.Status.ScheduleState)
|
||||||
|
// The first run should have been started.
|
||||||
|
assert.Equal(t, 1, len(app.Status.PastRunNames))
|
||||||
|
assert.False(t, app.Status.LastRun.IsZero())
|
||||||
|
assert.True(t, app.Status.NextRun.After(app.Status.LastRun.Time))
|
||||||
|
run, _ := c.crdClient.SparkoperatorV1alpha1().SparkApplications(app.Namespace).Get(app.Status.PastRunNames[0],
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.NotNil(t, run)
|
||||||
|
|
||||||
|
informer.GetIndexer().Add(app)
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(app.Namespace).Update(app)
|
||||||
|
// This sync should not start any run.
|
||||||
|
err = c.syncScheduledSparkApplication(key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app, _ = c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(app.Namespace).Get(app.Name,
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.Equal(t, v1alpha1.ScheduledState, app.Status.ScheduleState)
|
||||||
|
// Next run is not due, so there's still only one past run.
|
||||||
|
assert.Equal(t, 1, len(app.Status.PastRunNames))
|
||||||
|
|
||||||
|
informer.GetIndexer().Add(app)
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(app.Namespace).Update(app)
|
||||||
|
// Advance the clock to trigger the next run.
|
||||||
|
clk.SetTime(app.Status.NextRun.Time.Add(5 * time.Second))
|
||||||
|
// This sync should start the second run.
|
||||||
|
err = c.syncScheduledSparkApplication(key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app, _ = c.crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(app.Namespace).Get(app.Name,
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.Equal(t, v1alpha1.ScheduledState, app.Status.ScheduleState)
|
||||||
|
// The second run should have been started.
|
||||||
|
assert.Equal(t, 2, len(app.Status.PastRunNames))
|
||||||
|
run, _ = c.crdClient.SparkoperatorV1alpha1().SparkApplications(app.Namespace).Get(app.Status.PastRunNames[0],
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.NotNil(t, run)
|
||||||
|
run, _ = c.crdClient.SparkoperatorV1alpha1().SparkApplications(app.Namespace).Get(app.Status.PastRunNames[1],
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.NotNil(t, run)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShouldStartNextRun(t *testing.T) {
|
||||||
|
app := &v1alpha1.ScheduledSparkApplication{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Namespace: "default",
|
||||||
|
Name: "test-app",
|
||||||
|
},
|
||||||
|
Spec: v1alpha1.ScheduledSparkApplicationSpec{
|
||||||
|
Schedule: "@every 1m",
|
||||||
|
},
|
||||||
|
Status: v1alpha1.ScheduledSparkApplicationStatus{
|
||||||
|
PastRunNames: []string{"run1"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c, _, _ := newFakeController(app)
|
||||||
|
|
||||||
|
run1 := &v1alpha1.SparkApplication{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Namespace: app.Namespace,
|
||||||
|
Name: "run1",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().SparkApplications(run1.Namespace).Create(run1)
|
||||||
|
|
||||||
|
// ConcurrencyAllow with a running run.
|
||||||
|
run1.Status.AppState.State = v1alpha1.RunningState
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().SparkApplications(run1.Namespace).Update(run1)
|
||||||
|
app.Spec.ConcurrencyPolicy = v1alpha1.ConcurrencyAllow
|
||||||
|
ok, _ := c.shouldStartNextRun(app)
|
||||||
|
assert.True(t, ok)
|
||||||
|
|
||||||
|
// ConcurrencyForbid with a running run.
|
||||||
|
app.Spec.ConcurrencyPolicy = v1alpha1.ConcurrencyForbid
|
||||||
|
ok, _ = c.shouldStartNextRun(app)
|
||||||
|
assert.False(t, ok)
|
||||||
|
// ConcurrencyForbid with a completed run.
|
||||||
|
run1.Status.AppState.State = v1alpha1.CompletedState
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().SparkApplications(run1.Namespace).Update(run1)
|
||||||
|
ok, _ = c.shouldStartNextRun(app)
|
||||||
|
assert.True(t, ok)
|
||||||
|
|
||||||
|
// ConcurrencyReplace with a completed run.
|
||||||
|
app.Spec.ConcurrencyPolicy = v1alpha1.ConcurrencyReplace
|
||||||
|
ok, _ = c.shouldStartNextRun(app)
|
||||||
|
assert.True(t, ok)
|
||||||
|
// ConcurrencyReplace with a running run.
|
||||||
|
run1.Status.AppState.State = v1alpha1.RunningState
|
||||||
|
c.crdClient.SparkoperatorV1alpha1().SparkApplications(run1.Namespace).Update(run1)
|
||||||
|
ok, _ = c.shouldStartNextRun(app)
|
||||||
|
assert.True(t, ok)
|
||||||
|
// The previous running run should have been deleted.
|
||||||
|
existing, _ := c.crdClient.SparkoperatorV1alpha1().SparkApplications(run1.Namespace).Get(run1.Name,
|
||||||
|
metav1.GetOptions{})
|
||||||
|
assert.Nil(t, existing)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartNextRun(t *testing.T) {
|
||||||
|
app := &v1alpha1.ScheduledSparkApplication{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Namespace: "default",
|
||||||
|
Name: "test-app",
|
||||||
|
},
|
||||||
|
Spec: v1alpha1.ScheduledSparkApplicationSpec{
|
||||||
|
Schedule: "@every 1m",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c, _, _ := newFakeController(app)
|
||||||
|
|
||||||
|
schedule, _ := cron.ParseStandard(app.Spec.Schedule)
|
||||||
|
status := app.Status.DeepCopy()
|
||||||
|
|
||||||
|
err := c.startNextRun(app, status, schedule)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, 1, len(status.PastRunNames))
|
||||||
|
assert.True(t, status.NextRun.After(status.LastRun.Time))
|
||||||
|
// Check the first run.
|
||||||
|
firstRunName := status.PastRunNames[0]
|
||||||
|
run, _ := c.crdClient.SparkoperatorV1alpha1().SparkApplications(app.Namespace).Get(firstRunName, metav1.GetOptions{})
|
||||||
|
assert.NotNil(t, run)
|
||||||
|
|
||||||
|
err = c.startNextRun(app, status, schedule)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, 1, len(status.PastRunNames))
|
||||||
|
assert.True(t, status.NextRun.After(status.LastRun.Time))
|
||||||
|
// Check the second run.
|
||||||
|
secondRunName := status.PastRunNames[0]
|
||||||
|
run, _ = c.crdClient.SparkoperatorV1alpha1().SparkApplications(app.Namespace).Get(secondRunName, metav1.GetOptions{})
|
||||||
|
assert.NotNil(t, run)
|
||||||
|
// The second run should have a different name.
|
||||||
|
assert.NotEqual(t, secondRunName, firstRunName)
|
||||||
|
// The first run should have been deleted.
|
||||||
|
run, _ = c.crdClient.SparkoperatorV1alpha1().SparkApplications(app.Namespace).Get(firstRunName, metav1.GetOptions{})
|
||||||
|
assert.Nil(t, run)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeController(
|
||||||
|
apps ...*v1alpha1.ScheduledSparkApplication) (*Controller, cache.SharedIndexInformer, *clock.FakeClock) {
|
||||||
|
crdClient := crdclientfake.NewSimpleClientset()
|
||||||
|
kubeClient := kubeclientfake.NewSimpleClientset()
|
||||||
|
apiExtensionsClient := apiextensionsfake.NewSimpleClientset()
|
||||||
|
informerFactory := crdinformers.NewSharedInformerFactory(crdClient, 1*time.Second)
|
||||||
|
clk := clock.NewFakeClock(time.Now())
|
||||||
|
controller := NewController(crdClient, kubeClient, apiExtensionsClient, informerFactory, clk)
|
||||||
|
|
||||||
|
informer := informerFactory.Sparkoperator().V1alpha1().ScheduledSparkApplications().Informer()
|
||||||
|
for _, app := range apps {
|
||||||
|
crdClient.SparkoperatorV1alpha1().ScheduledSparkApplications(app.Namespace).Create(app)
|
||||||
|
informer.GetIndexer().Add(app)
|
||||||
|
}
|
||||||
|
|
||||||
|
return controller, informer, clk
|
||||||
|
}
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -42,7 +42,6 @@ import (
|
||||||
crdscheme "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned/scheme"
|
crdscheme "k8s.io/spark-on-k8s-operator/pkg/client/clientset/versioned/scheme"
|
||||||
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
||||||
crdlisters "k8s.io/spark-on-k8s-operator/pkg/client/listers/sparkoperator.k8s.io/v1alpha1"
|
crdlisters "k8s.io/spark-on-k8s-operator/pkg/client/listers/sparkoperator.k8s.io/v1alpha1"
|
||||||
"k8s.io/spark-on-k8s-operator/pkg/crd"
|
|
||||||
"k8s.io/spark-on-k8s-operator/pkg/util"
|
"k8s.io/spark-on-k8s-operator/pkg/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -58,8 +57,8 @@ var (
|
||||||
keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
|
keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
|
||||||
)
|
)
|
||||||
|
|
||||||
// SparkApplicationController manages instances of SparkApplication.
|
// Controller manages instances of SparkApplication.
|
||||||
type SparkApplicationController struct {
|
type Controller struct {
|
||||||
crdClient crdclientset.Interface
|
crdClient crdclientset.Interface
|
||||||
kubeClient clientset.Interface
|
kubeClient clientset.Interface
|
||||||
extensionsClient apiextensionsclient.Interface
|
extensionsClient apiextensionsclient.Interface
|
||||||
|
|
@ -73,13 +72,13 @@ type SparkApplicationController struct {
|
||||||
podStateReportingChan <-chan interface{}
|
podStateReportingChan <-chan interface{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new SparkApplicationController.
|
// NewController creates a new Controller.
|
||||||
func New(
|
func NewController(
|
||||||
crdClient crdclientset.Interface,
|
crdClient crdclientset.Interface,
|
||||||
kubeClient clientset.Interface,
|
kubeClient clientset.Interface,
|
||||||
extensionsClient apiextensionsclient.Interface,
|
extensionsClient apiextensionsclient.Interface,
|
||||||
informerFactory crdinformers.SharedInformerFactory,
|
informerFactory crdinformers.SharedInformerFactory,
|
||||||
submissionRunnerWorkers int) *SparkApplicationController {
|
submissionRunnerWorkers int) *Controller {
|
||||||
crdscheme.AddToScheme(scheme.Scheme)
|
crdscheme.AddToScheme(scheme.Scheme)
|
||||||
|
|
||||||
eventBroadcaster := record.NewBroadcaster()
|
eventBroadcaster := record.NewBroadcaster()
|
||||||
|
|
@ -99,7 +98,7 @@ func newSparkApplicationController(
|
||||||
extensionsClient apiextensionsclient.Interface,
|
extensionsClient apiextensionsclient.Interface,
|
||||||
informerFactory crdinformers.SharedInformerFactory,
|
informerFactory crdinformers.SharedInformerFactory,
|
||||||
eventRecorder record.EventRecorder,
|
eventRecorder record.EventRecorder,
|
||||||
submissionRunnerWorkers int) *SparkApplicationController {
|
submissionRunnerWorkers int) *Controller {
|
||||||
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(),
|
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(),
|
||||||
"spark-application-controller")
|
"spark-application-controller")
|
||||||
|
|
||||||
|
|
@ -109,7 +108,7 @@ func newSparkApplicationController(
|
||||||
runner := newSparkSubmitRunner(submissionRunnerWorkers, appStateReportingChan)
|
runner := newSparkSubmitRunner(submissionRunnerWorkers, appStateReportingChan)
|
||||||
sparkPodMonitor := newSparkPodMonitor(kubeClient, podStateReportingChan)
|
sparkPodMonitor := newSparkPodMonitor(kubeClient, podStateReportingChan)
|
||||||
|
|
||||||
controller := &SparkApplicationController{
|
controller := &Controller{
|
||||||
crdClient: crdClient,
|
crdClient: crdClient,
|
||||||
kubeClient: kubeClient,
|
kubeClient: kubeClient,
|
||||||
extensionsClient: extensionsClient,
|
extensionsClient: extensionsClient,
|
||||||
|
|
@ -133,17 +132,11 @@ func newSparkApplicationController(
|
||||||
return controller
|
return controller
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start starts the SparkApplicationController by registering a watcher for SparkApplication objects.
|
// Start starts the Controller by registering a watcher for SparkApplication objects.
|
||||||
func (s *SparkApplicationController) Start(workers int, stopCh <-chan struct{}) error {
|
func (c *Controller) Start(workers int, stopCh <-chan struct{}) error {
|
||||||
glog.Info("Starting the SparkApplication controller")
|
glog.Info("Starting the SparkApplication controller")
|
||||||
|
|
||||||
glog.Infof("Creating CustomResourceDefinition %s", crd.FullName)
|
if !cache.WaitForCacheSync(stopCh, c.cacheSynced) {
|
||||||
err := crd.CreateCRD(s.extensionsClient)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create CustomResourceDefinition %s: %v", crd.FullName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !cache.WaitForCacheSync(stopCh, s.cacheSynced) {
|
|
||||||
return fmt.Errorf("timed out waiting for cache to sync")
|
return fmt.Errorf("timed out waiting for cache to sync")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -151,40 +144,40 @@ func (s *SparkApplicationController) Start(workers int, stopCh <-chan struct{})
|
||||||
for i := 0; i < workers; i++ {
|
for i := 0; i < workers; i++ {
|
||||||
// runWorker will loop until "something bad" happens. Until will then rekick
|
// runWorker will loop until "something bad" happens. Until will then rekick
|
||||||
// the worker after one second.
|
// the worker after one second.
|
||||||
go wait.Until(s.runWorker, time.Second, stopCh)
|
go wait.Until(c.runWorker, time.Second, stopCh)
|
||||||
}
|
}
|
||||||
|
|
||||||
go s.runner.run(stopCh)
|
go c.runner.run(stopCh)
|
||||||
go s.sparkPodMonitor.run(stopCh)
|
go c.sparkPodMonitor.run(stopCh)
|
||||||
|
|
||||||
go s.processAppStateUpdates()
|
go c.processAppStateUpdates()
|
||||||
go s.processPodStateUpdates()
|
go c.processPodStateUpdates()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the controller.
|
// Stop stops the controller.
|
||||||
func (s *SparkApplicationController) Stop() {
|
func (c *Controller) Stop() {
|
||||||
glog.Info("Stopping the SparkApplication controller")
|
glog.Info("Stopping the SparkApplication controller")
|
||||||
s.queue.ShutDown()
|
c.queue.ShutDown()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Callback function called when a new SparkApplication object gets created.
|
// Callback function called when a new SparkApplication object gets created.
|
||||||
func (s *SparkApplicationController) onAdd(obj interface{}) {
|
func (c *Controller) onAdd(obj interface{}) {
|
||||||
app := obj.(*v1alpha1.SparkApplication)
|
app := obj.(*v1alpha1.SparkApplication)
|
||||||
if shouldSubmit(app) {
|
if shouldSubmit(app) {
|
||||||
glog.Infof("SparkApplication %s was added, enqueueing it for submission", app.Name)
|
glog.Infof("SparkApplication %c was added, enqueueing it for submission", app.Name)
|
||||||
s.enqueue(app)
|
c.enqueue(app)
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeNormal,
|
apiv1.EventTypeNormal,
|
||||||
"SparkApplicationAdded",
|
"SparkApplicationAdded",
|
||||||
"SparkApplication %s was added, enqueued it for submission",
|
"SparkApplication %c was added, enqueued it for submission",
|
||||||
app.Name)
|
app.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) onUpdate(oldObj, newObj interface{}) {
|
func (c *Controller) onUpdate(oldObj, newObj interface{}) {
|
||||||
oldApp := oldObj.(*v1alpha1.SparkApplication)
|
oldApp := oldObj.(*v1alpha1.SparkApplication)
|
||||||
newApp := newObj.(*v1alpha1.SparkApplication)
|
newApp := newObj.(*v1alpha1.SparkApplication)
|
||||||
|
|
||||||
|
|
@ -194,13 +187,13 @@ func (s *SparkApplicationController) onUpdate(oldObj, newObj interface{}) {
|
||||||
|
|
||||||
if oldApp.Status.DriverInfo.PodName != "" {
|
if oldApp.Status.DriverInfo.PodName != "" {
|
||||||
// Clear the application ID if the driver pod of the old application is to be deleted. This is important as
|
// Clear the application ID if the driver pod of the old application is to be deleted. This is important as
|
||||||
// otherwise deleting the driver pod of the old application if it's still running will result in a driver state
|
// otherwise deleting the driver pod of the old application if it'c still running will result in a driver state
|
||||||
// update by the sparkPodMonitor with the driver pod phase set to PodFailed. This may lead to a restart of the
|
// update by the sparkPodMonitor with the driver pod phase set to PodFailed. This may lead to a restart of the
|
||||||
// application if it's subject to a restart because of the failure state update of the driver pod. Clearing the
|
// application if it'c subject to a restart because of the failure state update of the driver pod. Clearing the
|
||||||
// application ID causes the state update regarding the old driver pod to be ignored in
|
// application ID causes the state update regarding the old driver pod to be ignored in
|
||||||
// processSingleDriverStateUpdate because of mismatched application IDs and as a consequence no restart to be
|
// processSingleDriverStateUpdate because of mismatched application IDs and as a consequence no restart to be
|
||||||
// triggered. This prevents the application to be submitted twice: one from the restart and one from below.
|
// triggered. This prevents the application to be submitted twice: one from the restart and one from below.
|
||||||
s.updateSparkApplicationStatusWithRetries(newApp, func(status *v1alpha1.SparkApplicationStatus) {
|
c.updateSparkApplicationStatusWithRetries(newApp, func(status *v1alpha1.SparkApplicationStatus) {
|
||||||
status.AppID = ""
|
status.AppID = ""
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -208,23 +201,23 @@ func (s *SparkApplicationController) onUpdate(oldObj, newObj interface{}) {
|
||||||
// Delete the driver pod and UI service of the old application. Note that deleting the driver pod kills the
|
// Delete the driver pod and UI service of the old application. Note that deleting the driver pod kills the
|
||||||
// application if it is still running. Skip submitting the new application if cleanup for the old application
|
// application if it is still running. Skip submitting the new application if cleanup for the old application
|
||||||
// failed to avoid potentially running both the old and new applications at the same time.
|
// failed to avoid potentially running both the old and new applications at the same time.
|
||||||
if err := s.deleteDriverAndUIService(oldApp, true); err != nil {
|
if err := c.deleteDriverAndUIService(oldApp, true); err != nil {
|
||||||
glog.Error(err)
|
glog.Error(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
glog.Infof("SparkApplication %s was updated, enqueueing it for submission", newApp.Name)
|
glog.Infof("SparkApplication %c was updated, enqueueing it for submission", newApp.Name)
|
||||||
s.enqueue(newApp)
|
c.enqueue(newApp)
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
newApp,
|
newApp,
|
||||||
apiv1.EventTypeNormal,
|
apiv1.EventTypeNormal,
|
||||||
"SparkApplicationUpdated",
|
"SparkApplicationUpdated",
|
||||||
"SparkApplication %s was updated, enqueued it for submission",
|
"SparkApplication %c was updated, enqueued it for submission",
|
||||||
newApp.Name)
|
newApp.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) onDelete(obj interface{}) {
|
func (c *Controller) onDelete(obj interface{}) {
|
||||||
s.dequeue(obj)
|
c.dequeue(obj)
|
||||||
|
|
||||||
var app *v1alpha1.SparkApplication
|
var app *v1alpha1.SparkApplication
|
||||||
switch obj.(type) {
|
switch obj.(type) {
|
||||||
|
|
@ -236,34 +229,34 @@ func (s *SparkApplicationController) onDelete(obj interface{}) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if app != nil {
|
if app != nil {
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeNormal,
|
apiv1.EventTypeNormal,
|
||||||
"SparkApplicationDeleted",
|
"SparkApplicationDeleted",
|
||||||
"SparkApplication %s was deleted",
|
"SparkApplication %c was deleted",
|
||||||
app.Name)
|
app.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runWorker runs a single controller worker.
|
// runWorker runs a single controller worker.
|
||||||
func (s *SparkApplicationController) runWorker() {
|
func (c *Controller) runWorker() {
|
||||||
defer utilruntime.HandleCrash()
|
defer utilruntime.HandleCrash()
|
||||||
for s.processNextItem() {
|
for c.processNextItem() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) processNextItem() bool {
|
func (c *Controller) processNextItem() bool {
|
||||||
key, quit := s.queue.Get()
|
key, quit := c.queue.Get()
|
||||||
if quit {
|
if quit {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
defer s.queue.Done(key)
|
defer c.queue.Done(key)
|
||||||
|
|
||||||
err := s.syncSparkApplication(key.(string))
|
err := c.syncSparkApplication(key.(string))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// Successfully processed the key or the key was not found so tell the queue to stop tracking
|
// Successfully processed the key or the key was not found so tell the queue to stop tracking
|
||||||
// history for your key. This will reset things like failure counts for per-item rate limiting.
|
// history for your key. This will reset things like failure counts for per-item rate limiting.
|
||||||
s.queue.Forget(key)
|
c.queue.Forget(key)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -274,29 +267,29 @@ func (s *SparkApplicationController) processNextItem() bool {
|
||||||
// to avoid hot-looping on particular items (they're probably still not going to work right away)
|
// to avoid hot-looping on particular items (they're probably still not going to work right away)
|
||||||
// and overall controller protection (everything I've done is broken, this controller needs to
|
// and overall controller protection (everything I've done is broken, this controller needs to
|
||||||
// calm down or it can starve other useful work) cases.
|
// calm down or it can starve other useful work) cases.
|
||||||
s.queue.AddRateLimited(key)
|
c.queue.AddRateLimited(key)
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) syncSparkApplication(key string) error {
|
func (c *Controller) syncSparkApplication(key string) error {
|
||||||
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get the namespace and name from key %s: %v", key, err)
|
return fmt.Errorf("failed to get the namespace and name from key %c: %v", key, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
app, err := s.getSparkApplication(namespace, name)
|
app, err := c.getSparkApplication(namespace, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = s.createSubmission(app)
|
err = c.createSubmission(app)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeWarning,
|
apiv1.EventTypeWarning,
|
||||||
"SparkApplicationSubmissionCreationFailed",
|
"SparkApplicationSubmissionCreationFailed",
|
||||||
"failed to create a submission for SparkApplication %s: %s",
|
"failed to create a submission for SparkApplication %c: %c",
|
||||||
app.Name,
|
app.Name,
|
||||||
err.Error())
|
err.Error())
|
||||||
return err
|
return err
|
||||||
|
|
@ -306,22 +299,22 @@ func (s *SparkApplicationController) syncSparkApplication(key string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// createSubmission creates a new submission for the given SparkApplication and send it to the submission runner.
|
// createSubmission creates a new submission for the given SparkApplication and send it to the submission runner.
|
||||||
func (s *SparkApplicationController) createSubmission(app *v1alpha1.SparkApplication) error {
|
func (c *Controller) createSubmission(app *v1alpha1.SparkApplication) error {
|
||||||
appStatus := v1alpha1.SparkApplicationStatus{
|
appStatus := v1alpha1.SparkApplicationStatus{
|
||||||
AppID: buildAppID(app),
|
AppID: buildAppID(app),
|
||||||
AppState: v1alpha1.ApplicationState{
|
AppState: v1alpha1.ApplicationState{
|
||||||
State: v1alpha1.NewState,
|
State: v1alpha1.NewState,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
name, port, err := createSparkUIService(app, appStatus.AppID, s.kubeClient)
|
name, port, err := createSparkUIService(app, appStatus.AppID, c.kubeClient)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to create a UI service for SparkApplication %s: %v", app.Name, err)
|
glog.Errorf("failed to create a UI service for SparkApplication %c: %v", app.Name, err)
|
||||||
} else {
|
} else {
|
||||||
appStatus.DriverInfo.WebUIServiceName = name
|
appStatus.DriverInfo.WebUIServiceName = name
|
||||||
appStatus.DriverInfo.WebUIPort = port
|
appStatus.DriverInfo.WebUIPort = port
|
||||||
}
|
}
|
||||||
|
|
||||||
updatedApp := s.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
updatedApp := c.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
||||||
*status = v1alpha1.SparkApplicationStatus{}
|
*status = v1alpha1.SparkApplicationStatus{}
|
||||||
appStatus.DeepCopyInto(status)
|
appStatus.DeepCopyInto(status)
|
||||||
})
|
})
|
||||||
|
|
@ -332,42 +325,41 @@ func (s *SparkApplicationController) createSubmission(app *v1alpha1.SparkApplica
|
||||||
submissionCmdArgs, err := buildSubmissionCommandArgs(updatedApp)
|
submissionCmdArgs, err := buildSubmissionCommandArgs(updatedApp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"failed to build the submission command for SparkApplication %s: %v",
|
"failed to build the submission command for SparkApplication %c: %v",
|
||||||
updatedApp.Name,
|
updatedApp.Name,
|
||||||
err)
|
err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.runner.submit(newSubmission(submissionCmdArgs, updatedApp))
|
c.runner.submit(newSubmission(submissionCmdArgs, updatedApp))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) processPodStateUpdates() {
|
func (c *Controller) processPodStateUpdates() {
|
||||||
for update := range s.podStateReportingChan {
|
for update := range c.podStateReportingChan {
|
||||||
switch update.(type) {
|
switch update.(type) {
|
||||||
case *driverStateUpdate:
|
case *driverStateUpdate:
|
||||||
updatedApp := s.processSingleDriverStateUpdate(update.(*driverStateUpdate))
|
updatedApp := c.processSingleDriverStateUpdate(update.(*driverStateUpdate))
|
||||||
if updatedApp != nil && shouldRestart(updatedApp) {
|
if updatedApp != nil && shouldRestart(updatedApp) {
|
||||||
s.handleRestart(updatedApp)
|
c.handleRestart(updatedApp)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
case *executorStateUpdate:
|
case *executorStateUpdate:
|
||||||
s.processSingleExecutorStateUpdate(update.(*executorStateUpdate))
|
c.processSingleExecutorStateUpdate(update.(*executorStateUpdate))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) processSingleDriverStateUpdate(
|
func (c *Controller) processSingleDriverStateUpdate(update *driverStateUpdate) *v1alpha1.SparkApplication {
|
||||||
update *driverStateUpdate) *v1alpha1.SparkApplication {
|
|
||||||
glog.V(2).Infof(
|
glog.V(2).Infof(
|
||||||
"Received driver state update for SparkApplication %s in namespace %s with phase %s",
|
"Received driver state update for SparkApplication %c in namespace %c with phase %c",
|
||||||
update.appName, update.appNamespace, update.podPhase)
|
update.appName, update.appNamespace, update.podPhase)
|
||||||
|
|
||||||
app, err := s.getSparkApplication(update.appNamespace, update.appName)
|
app, err := c.getSparkApplication(update.appNamespace, update.appName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Update may be the result of pod deletion due to deletion of the owning SparkApplication object.
|
// Update may be the result of pod deletion due to deletion of the owning SparkApplication object.
|
||||||
// Ignore the error if the owning SparkApplication object does not exist.
|
// Ignore the error if the owning SparkApplication object does not exist.
|
||||||
if !errors.IsNotFound(err) {
|
if !errors.IsNotFound(err) {
|
||||||
glog.Errorf("failed to get SparkApplication %s in namespace %s from the store: %v", update.appName,
|
glog.Errorf("failed to get SparkApplication %c in namespace %c from the store: %v", update.appName,
|
||||||
update.appNamespace, err)
|
update.appNamespace, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
@ -381,26 +373,26 @@ func (s *SparkApplicationController) processSingleDriverStateUpdate(
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
s.recordDriverEvent(app, update.podPhase, update.podName)
|
c.recordDriverEvent(app, update.podPhase, update.podName)
|
||||||
|
|
||||||
// The application state is solely based on the driver pod phase once the application is successfully
|
// The application state is solely based on the driver pod phase once the application is successfully
|
||||||
// submitted and the driver pod is created.
|
// submitted and the driver pod is created.
|
||||||
appState := driverPodPhaseToApplicationState(update.podPhase)
|
appState := driverPodPhaseToApplicationState(update.podPhase)
|
||||||
if isAppTerminated(appState) {
|
if isAppTerminated(appState) {
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeNormal,
|
apiv1.EventTypeNormal,
|
||||||
"SparkApplicationTerminated",
|
"SparkApplicationTerminated",
|
||||||
"SparkApplication %s terminated with state: %v",
|
"SparkApplication %c terminated with state: %v",
|
||||||
update.appName,
|
update.appName,
|
||||||
appState)
|
appState)
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
return c.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
||||||
status.DriverInfo.PodName = update.podName
|
status.DriverInfo.PodName = update.podName
|
||||||
if update.nodeName != "" {
|
if update.nodeName != "" {
|
||||||
if nodeIP := s.getNodeExternalIP(update.nodeName); nodeIP != "" {
|
if nodeIP := c.getNodeExternalIP(update.nodeName); nodeIP != "" {
|
||||||
status.DriverInfo.WebUIAddress = fmt.Sprintf("%s:%d", nodeIP,
|
status.DriverInfo.WebUIAddress = fmt.Sprintf("%c:%d", nodeIP,
|
||||||
status.DriverInfo.WebUIPort)
|
status.DriverInfo.WebUIPort)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -412,53 +404,53 @@ func (s *SparkApplicationController) processSingleDriverStateUpdate(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) processAppStateUpdates() {
|
func (c *Controller) processAppStateUpdates() {
|
||||||
for update := range s.appStateReportingChan {
|
for update := range c.appStateReportingChan {
|
||||||
s.processSingleAppStateUpdate(update)
|
c.processSingleAppStateUpdate(update)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) processSingleAppStateUpdate(update *appStateUpdate) *v1alpha1.SparkApplication {
|
func (c *Controller) processSingleAppStateUpdate(update *appStateUpdate) *v1alpha1.SparkApplication {
|
||||||
app, err := s.getSparkApplication(update.namespace, update.name)
|
app, err := c.getSparkApplication(update.namespace, update.name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to get SparkApplication %s in namespace %s from the store: %v", update.name,
|
glog.Errorf("failed to get SparkApplication %c in namespace %c from the store: %v", update.name,
|
||||||
update.namespace, err)
|
update.namespace, err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
submissionRetries := app.Status.SubmissionRetries
|
submissionRetries := app.Status.SubmissionRetries
|
||||||
if update.state == v1alpha1.FailedSubmissionState {
|
if update.state == v1alpha1.FailedSubmissionState {
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeWarning,
|
apiv1.EventTypeWarning,
|
||||||
"SparkApplicationSubmissionFailed",
|
"SparkApplicationSubmissionFailed",
|
||||||
"SparkApplication %s failed submission: %s",
|
"SparkApplication %c failed submission: %c",
|
||||||
update.name,
|
update.name,
|
||||||
update.errorMessage)
|
update.errorMessage)
|
||||||
|
|
||||||
if shouldRetrySubmission(app) {
|
if shouldRetrySubmission(app) {
|
||||||
glog.Infof("Retrying submission of SparkApplication %s", update.name)
|
glog.Infof("Retrying submission of SparkApplication %c", update.name)
|
||||||
|
|
||||||
submissionRetries++
|
submissionRetries++
|
||||||
if app.Spec.SubmissionRetryInterval != nil {
|
if app.Spec.SubmissionRetryInterval != nil {
|
||||||
interval := time.Duration(*app.Spec.SubmissionRetryInterval) * time.Second
|
interval := time.Duration(*app.Spec.SubmissionRetryInterval) * time.Second
|
||||||
s.enqueueAfter(app, time.Duration(submissionRetries)*interval)
|
c.enqueueAfter(app, time.Duration(submissionRetries)*interval)
|
||||||
} else {
|
} else {
|
||||||
s.enqueue(app)
|
c.enqueue(app)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeNormal,
|
apiv1.EventTypeNormal,
|
||||||
"SparkApplicationSubmissionRetry",
|
"SparkApplicationSubmissionRetry",
|
||||||
"SparkApplication %s is scheduled for a submission retry",
|
"SparkApplication %c is scheduled for a submission retry",
|
||||||
update.name)
|
update.name)
|
||||||
} else {
|
} else {
|
||||||
glog.Infof("Not retrying submission of SparkApplication %s", update.name)
|
glog.Infof("Not retrying submission of SparkApplication %c", update.name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
return c.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
||||||
status.AppState.State = update.state
|
status.AppState.State = update.state
|
||||||
status.AppState.ErrorMessage = update.errorMessage
|
status.AppState.ErrorMessage = update.errorMessage
|
||||||
status.SubmissionRetries = submissionRetries
|
status.SubmissionRetries = submissionRetries
|
||||||
|
|
@ -468,18 +460,17 @@ func (s *SparkApplicationController) processSingleAppStateUpdate(update *appStat
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) processSingleExecutorStateUpdate(
|
func (c *Controller) processSingleExecutorStateUpdate(update *executorStateUpdate) *v1alpha1.SparkApplication {
|
||||||
update *executorStateUpdate) *v1alpha1.SparkApplication {
|
|
||||||
glog.V(2).Infof(
|
glog.V(2).Infof(
|
||||||
"Received state update of executor %s for SparkApplication %s in namespace %s with state %s",
|
"Received state update of executor %c for SparkApplication %c in namespace %c with state %c",
|
||||||
update.executorID, update.appName, update.appNamespace, update.state)
|
update.executorID, update.appName, update.appNamespace, update.state)
|
||||||
|
|
||||||
app, err := s.getSparkApplication(update.appNamespace, update.appName)
|
app, err := c.getSparkApplication(update.appNamespace, update.appName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Update may be the result of pod deletion due to deletion of the owning SparkApplication object.
|
// Update may be the result of pod deletion due to deletion of the owning SparkApplication object.
|
||||||
// Ignore the error if the owning SparkApplication object does not exist.
|
// Ignore the error if the owning SparkApplication object does not exist.
|
||||||
if !errors.IsNotFound(err) {
|
if !errors.IsNotFound(err) {
|
||||||
glog.Errorf("failed to get SparkApplication %s in namespace %s from the store: %v", update.appName,
|
glog.Errorf("failed to get SparkApplication %c in namespace %c from the store: %v", update.appName,
|
||||||
update.appNamespace, err)
|
update.appNamespace, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
@ -493,9 +484,9 @@ func (s *SparkApplicationController) processSingleExecutorStateUpdate(
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
s.recordExecutorEvent(app, update.state, update.podName)
|
c.recordExecutorEvent(app, update.state, update.podName)
|
||||||
|
|
||||||
return s.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
return c.updateSparkApplicationStatusWithRetries(app, func(status *v1alpha1.SparkApplicationStatus) {
|
||||||
if status.ExecutorState == nil {
|
if status.ExecutorState == nil {
|
||||||
status.ExecutorState = make(map[string]v1alpha1.ExecutorState)
|
status.ExecutorState = make(map[string]v1alpha1.ExecutorState)
|
||||||
}
|
}
|
||||||
|
|
@ -507,14 +498,14 @@ func (s *SparkApplicationController) processSingleExecutorStateUpdate(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) updateSparkApplicationStatusWithRetries(
|
func (c *Controller) updateSparkApplicationStatusWithRetries(
|
||||||
original *v1alpha1.SparkApplication,
|
original *v1alpha1.SparkApplication,
|
||||||
updateFunc func(*v1alpha1.SparkApplicationStatus)) *v1alpha1.SparkApplication {
|
updateFunc func(*v1alpha1.SparkApplicationStatus)) *v1alpha1.SparkApplication {
|
||||||
toUpdate := original.DeepCopy()
|
toUpdate := original.DeepCopy()
|
||||||
|
|
||||||
var lastUpdateErr error
|
var lastUpdateErr error
|
||||||
for i := 0; i < maximumUpdateRetries; i++ {
|
for i := 0; i < maximumUpdateRetries; i++ {
|
||||||
updated, err := s.tryUpdateStatus(original, toUpdate, updateFunc)
|
updated, err := c.tryUpdateStatus(original, toUpdate, updateFunc)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return updated
|
return updated
|
||||||
}
|
}
|
||||||
|
|
@ -523,22 +514,22 @@ func (s *SparkApplicationController) updateSparkApplicationStatusWithRetries(
|
||||||
// Failed update to the API server.
|
// Failed update to the API server.
|
||||||
// Get the latest version from the API server first and re-apply the update.
|
// Get the latest version from the API server first and re-apply the update.
|
||||||
name := toUpdate.Name
|
name := toUpdate.Name
|
||||||
toUpdate, err = s.crdClient.SparkoperatorV1alpha1().SparkApplications(toUpdate.Namespace).Get(name,
|
toUpdate, err = c.crdClient.SparkoperatorV1alpha1().SparkApplications(toUpdate.Namespace).Get(name,
|
||||||
metav1.GetOptions{})
|
metav1.GetOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to get SparkApplication %s: %v", name, err)
|
glog.Errorf("failed to get SparkApplication %c: %v", name, err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if lastUpdateErr != nil {
|
if lastUpdateErr != nil {
|
||||||
glog.Errorf("failed to update SparkApplication %s: %v", toUpdate.Name, lastUpdateErr)
|
glog.Errorf("failed to update SparkApplication %c: %v", toUpdate.Name, lastUpdateErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) tryUpdateStatus(
|
func (c *Controller) tryUpdateStatus(
|
||||||
original *v1alpha1.SparkApplication,
|
original *v1alpha1.SparkApplication,
|
||||||
toUpdate *v1alpha1.SparkApplication,
|
toUpdate *v1alpha1.SparkApplication,
|
||||||
updateFunc func(*v1alpha1.SparkApplicationStatus)) (*v1alpha1.SparkApplication, error) {
|
updateFunc func(*v1alpha1.SparkApplicationStatus)) (*v1alpha1.SparkApplication, error) {
|
||||||
|
|
@ -547,49 +538,48 @@ func (s *SparkApplicationController) tryUpdateStatus(
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.crdClient.SparkoperatorV1alpha1().SparkApplications(toUpdate.Namespace).Update(toUpdate)
|
return c.crdClient.SparkoperatorV1alpha1().SparkApplications(toUpdate.Namespace).Update(toUpdate)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) getSparkApplication(namespace string, name string) (
|
func (c *Controller) getSparkApplication(namespace string, name string) (*v1alpha1.SparkApplication, error) {
|
||||||
*v1alpha1.SparkApplication, error) {
|
return c.lister.SparkApplications(namespace).Get(name)
|
||||||
return s.lister.SparkApplications(namespace).Get(name)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) enqueue(obj interface{}) {
|
func (c *Controller) enqueue(obj interface{}) {
|
||||||
key, err := keyFunc(obj)
|
key, err := keyFunc(obj)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to get key for %v: %v", obj, err)
|
glog.Errorf("failed to get key for %v: %v", obj, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.queue.AddRateLimited(key)
|
c.queue.AddRateLimited(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) enqueueAfter(obj interface{}, after time.Duration) {
|
func (c *Controller) enqueueAfter(obj interface{}, after time.Duration) {
|
||||||
key, err := keyFunc(obj)
|
key, err := keyFunc(obj)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to get key for %v: %v", obj, err)
|
glog.Errorf("failed to get key for %v: %v", obj, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.queue.AddAfter(key, after)
|
c.queue.AddAfter(key, after)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) dequeue(obj interface{}) {
|
func (c *Controller) dequeue(obj interface{}) {
|
||||||
key, err := keyFunc(obj)
|
key, err := keyFunc(obj)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to get key for %v: %v", obj, err)
|
glog.Errorf("failed to get key for %v: %v", obj, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.queue.Forget(key)
|
c.queue.Forget(key)
|
||||||
s.queue.Done(key)
|
c.queue.Done(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) getNodeExternalIP(nodeName string) string {
|
func (c *Controller) getNodeExternalIP(nodeName string) string {
|
||||||
node, err := s.kubeClient.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
node, err := c.kubeClient.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("failed to get node %s", nodeName)
|
glog.Errorf("failed to get node %c", nodeName)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -603,14 +593,14 @@ func (s *SparkApplicationController) getNodeExternalIP(nodeName string) string {
|
||||||
|
|
||||||
// handleRestart handles application restart if the application has terminated and is subject to restart according to
|
// handleRestart handles application restart if the application has terminated and is subject to restart according to
|
||||||
// the restart policy.
|
// the restart policy.
|
||||||
func (s *SparkApplicationController) handleRestart(app *v1alpha1.SparkApplication) {
|
func (c *Controller) handleRestart(app *v1alpha1.SparkApplication) {
|
||||||
glog.Infof("SparkApplication %s failed or terminated, restarting it with RestartPolicy %s",
|
glog.Infof("SparkApplication %c failed or terminated, restarting it with RestartPolicy %c",
|
||||||
app.Name, app.Spec.RestartPolicy)
|
app.Name, app.Spec.RestartPolicy)
|
||||||
s.recorder.Eventf(
|
c.recorder.Eventf(
|
||||||
app,
|
app,
|
||||||
apiv1.EventTypeNormal,
|
apiv1.EventTypeNormal,
|
||||||
"SparkApplicationRestart",
|
"SparkApplicationRestart",
|
||||||
"SparkApplication %s is subject to restart",
|
"SparkApplication %c is subject to restart",
|
||||||
app.Name)
|
app.Name)
|
||||||
|
|
||||||
// Delete the old driver pod and UI service if necessary. Note that in case an error occurred here, we simply
|
// Delete the old driver pod and UI service if necessary. Note that in case an error occurred here, we simply
|
||||||
|
|
@ -618,38 +608,36 @@ func (s *SparkApplicationController) handleRestart(app *v1alpha1.SparkApplicatio
|
||||||
// terminated so failure to cleanup the old driver pod and UI service is not a blocker. Also note that because
|
// terminated so failure to cleanup the old driver pod and UI service is not a blocker. Also note that because
|
||||||
// deleting a already terminated driver pod won't trigger a driver state update by the sparkPodMonitor so won't
|
// deleting a already terminated driver pod won't trigger a driver state update by the sparkPodMonitor so won't
|
||||||
// cause repetitive restart handling.
|
// cause repetitive restart handling.
|
||||||
if err := s.deleteDriverAndUIService(app, false); err != nil {
|
if err := c.deleteDriverAndUIService(app, false); err != nil {
|
||||||
glog.Error(err)
|
glog.Error(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
//Enqueue the object for re-submission.
|
//Enqueue the object for re-submission.
|
||||||
s.enqueue(app)
|
c.enqueue(app)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) deleteDriverAndUIService(
|
func (c *Controller) deleteDriverAndUIService(app *v1alpha1.SparkApplication, waitForDriverDeletion bool) error {
|
||||||
app *v1alpha1.SparkApplication,
|
|
||||||
waitForDriverDeletion bool) error {
|
|
||||||
var zero int64
|
var zero int64
|
||||||
if app.Status.DriverInfo.PodName != "" {
|
if app.Status.DriverInfo.PodName != "" {
|
||||||
err := s.kubeClient.CoreV1().Pods(app.Namespace).Delete(app.Status.DriverInfo.PodName,
|
err := c.kubeClient.CoreV1().Pods(app.Namespace).Delete(app.Status.DriverInfo.PodName,
|
||||||
&metav1.DeleteOptions{GracePeriodSeconds: &zero})
|
&metav1.DeleteOptions{GracePeriodSeconds: &zero})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to delete old driver pod %s of SparkApplication %s: %v",
|
return fmt.Errorf("failed to delete old driver pod %c of SparkApplication %c: %v",
|
||||||
app.Status.DriverInfo.PodName, app.Name, err)
|
app.Status.DriverInfo.PodName, app.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if app.Status.DriverInfo.WebUIServiceName != "" {
|
if app.Status.DriverInfo.WebUIServiceName != "" {
|
||||||
err := s.kubeClient.CoreV1().Services(app.Namespace).Delete(app.Status.DriverInfo.WebUIServiceName,
|
err := c.kubeClient.CoreV1().Services(app.Namespace).Delete(app.Status.DriverInfo.WebUIServiceName,
|
||||||
&metav1.DeleteOptions{GracePeriodSeconds: &zero})
|
&metav1.DeleteOptions{GracePeriodSeconds: &zero})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to delete old web UI service %s of SparkApplication %s: %v",
|
return fmt.Errorf("failed to delete old web UI service %c of SparkApplication %c: %v",
|
||||||
app.Status.DriverInfo.WebUIServiceName, app.Name, err)
|
app.Status.DriverInfo.WebUIServiceName, app.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if waitForDriverDeletion {
|
if waitForDriverDeletion {
|
||||||
wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
|
wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
|
||||||
_, err := s.kubeClient.CoreV1().Pods(app.Namespace).Get(app.Status.DriverInfo.PodName, metav1.GetOptions{})
|
_, err := c.kubeClient.CoreV1().Pods(app.Namespace).Get(app.Status.DriverInfo.PodName, metav1.GetOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.IsNotFound(err) {
|
if errors.IsNotFound(err) {
|
||||||
return true, nil
|
return true, nil
|
||||||
|
|
@ -663,21 +651,21 @@ func (s *SparkApplicationController) deleteDriverAndUIService(
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) recordDriverEvent(
|
func (c *Controller) recordDriverEvent(
|
||||||
app *v1alpha1.SparkApplication, phase apiv1.PodPhase, name string) {
|
app *v1alpha1.SparkApplication, phase apiv1.PodPhase, name string) {
|
||||||
if phase == apiv1.PodSucceeded {
|
if phase == apiv1.PodSucceeded {
|
||||||
s.recorder.Eventf(app, apiv1.EventTypeNormal, "SparkDriverCompleted", "Driver %s completed", name)
|
c.recorder.Eventf(app, apiv1.EventTypeNormal, "SparkDriverCompleted", "Driver %s completed", name)
|
||||||
} else if phase == apiv1.PodFailed {
|
} else if phase == apiv1.PodFailed {
|
||||||
s.recorder.Eventf(app, apiv1.EventTypeWarning, "SparkDriverFailed", "Driver %s failed", name)
|
c.recorder.Eventf(app, apiv1.EventTypeWarning, "SparkDriverFailed", "Driver %s failed", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SparkApplicationController) recordExecutorEvent(
|
func (c *Controller) recordExecutorEvent(
|
||||||
app *v1alpha1.SparkApplication, state v1alpha1.ExecutorState, name string) {
|
app *v1alpha1.SparkApplication, state v1alpha1.ExecutorState, name string) {
|
||||||
if state == v1alpha1.ExecutorCompletedState {
|
if state == v1alpha1.ExecutorCompletedState {
|
||||||
s.recorder.Eventf(app, apiv1.EventTypeNormal, "SparkExecutorCompleted", "Executor %s completed", name)
|
c.recorder.Eventf(app, apiv1.EventTypeNormal, "SparkExecutorCompleted", "Executor %s completed", name)
|
||||||
} else if state == v1alpha1.ExecutorFailedState {
|
} else if state == v1alpha1.ExecutorFailedState {
|
||||||
s.recorder.Eventf(app, apiv1.EventTypeWarning, "SparkExecutorFailed", "Executor %s failed", name)
|
c.recorder.Eventf(app, apiv1.EventTypeWarning, "SparkExecutorFailed", "Executor %s failed", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
|
|
@ -37,7 +37,7 @@ import (
|
||||||
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
crdinformers "k8s.io/spark-on-k8s-operator/pkg/client/informers/externalversions"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newFakeController(apps ...*v1alpha1.SparkApplication) (*SparkApplicationController, *record.FakeRecorder) {
|
func newFakeController(apps ...*v1alpha1.SparkApplication) (*Controller, *record.FakeRecorder) {
|
||||||
crdclientfake.AddToScheme(scheme.Scheme)
|
crdclientfake.AddToScheme(scheme.Scheme)
|
||||||
crdClient := crdclientfake.NewSimpleClientset()
|
crdClient := crdclientfake.NewSimpleClientset()
|
||||||
kubeClient := kubeclientfake.NewSimpleClientset()
|
kubeClient := kubeclientfake.NewSimpleClientset()
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"reflect"
|
"reflect"
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
@ -14,4 +14,4 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package controller
|
package sparkapplication
|
||||||
|
|
@ -17,4 +17,5 @@ limitations under the License.
|
||||||
package crd
|
package crd
|
||||||
|
|
||||||
// Package crd contains the definition of and code to work with the CustomResourceDefinition (CRD)
|
// Package crd contains the definition of and code to work with the CustomResourceDefinition (CRD)
|
||||||
// for SparkApplication and a client to perform CRUD operations on instances of SparkApplication.
|
// for SparkApplication and ScheduledSparkApplication and a client to perform CRUD operations on
|
||||||
|
// instances of SparkApplication and ScheduledSparkApplication.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,220 @@
|
||||||
|
/*
|
||||||
|
Copyright 2017 Google LLC
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package scheduledsparkapplication
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/golang/glog"
|
||||||
|
|
||||||
|
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||||
|
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||||
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/errors"
|
||||||
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
|
|
||||||
|
"k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io"
|
||||||
|
"k8s.io/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CRD metadata.
|
||||||
|
const (
|
||||||
|
Plural = "scheduledsparkapplications"
|
||||||
|
Singular = "scheduledsparkapplication"
|
||||||
|
ShortName = "scheduledsparkapp"
|
||||||
|
Group = sparkoperator.GroupName
|
||||||
|
Version = "v1alpha1"
|
||||||
|
FullName = Plural + "." + Group
|
||||||
|
)
|
||||||
|
|
||||||
|
// CreateCRD creates a Kubernetes CustomResourceDefinition (CRD) for SparkApplication.
|
||||||
|
// An error is returned if it fails to create the CustomResourceDefinition before it times out.
|
||||||
|
func CreateCRD(clientset apiextensionsclient.Interface) error {
|
||||||
|
// The CustomResourceDefinition is not found, create it now.
|
||||||
|
sparkAppCrd := &apiextensionsv1beta1.CustomResourceDefinition{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: FullName,
|
||||||
|
},
|
||||||
|
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
|
||||||
|
Group: Group,
|
||||||
|
Version: Version,
|
||||||
|
Scope: apiextensionsv1beta1.NamespaceScoped,
|
||||||
|
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
|
||||||
|
Plural: Plural,
|
||||||
|
Singular: Singular,
|
||||||
|
ShortNames: []string{ShortName},
|
||||||
|
Kind: reflect.TypeOf(v1alpha1.ScheduledSparkApplication{}).Name(),
|
||||||
|
},
|
||||||
|
Validation: getCustomResourceValidation(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(sparkAppCrd)
|
||||||
|
if err != nil {
|
||||||
|
if apierrors.IsAlreadyExists(err) {
|
||||||
|
glog.Warningf("CustomResourceDefinition %s already exists", FullName)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for the CustomResourceDefinition to become registered.
|
||||||
|
err = waitForCRDEstablishment(clientset)
|
||||||
|
// Try deleting the CustomResourceDefinition if it fails to be registered on time.
|
||||||
|
if err != nil {
|
||||||
|
deleteErr := deleteCRD(clientset)
|
||||||
|
if deleteErr != nil {
|
||||||
|
return errors.NewAggregate([]error{err, deleteErr})
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteCRD(clientset apiextensionsclient.Interface) error {
|
||||||
|
var zero int64
|
||||||
|
err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(FullName,
|
||||||
|
&metav1.DeleteOptions{GracePeriodSeconds: &zero})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getCRD(clientset apiextensionsclient.Interface) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
|
||||||
|
return clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(FullName, metav1.GetOptions{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForCRDEstablishment waits for the CRD to be registered and established until it times out.
|
||||||
|
func waitForCRDEstablishment(clientset apiextensionsclient.Interface) error {
|
||||||
|
return wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
|
||||||
|
sparkAppCrd, err := getCRD(clientset)
|
||||||
|
for _, cond := range sparkAppCrd.Status.Conditions {
|
||||||
|
switch cond.Type {
|
||||||
|
case apiextensionsv1beta1.Established:
|
||||||
|
if cond.Status == apiextensionsv1beta1.ConditionTrue {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
case apiextensionsv1beta1.NamesAccepted:
|
||||||
|
if cond.Status == apiextensionsv1beta1.ConditionFalse {
|
||||||
|
fmt.Printf("Name conflict: %v\n", cond.Reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func getCustomResourceValidation() *apiextensionsv1beta1.CustomResourceValidation {
|
||||||
|
return &apiextensionsv1beta1.CustomResourceValidation{
|
||||||
|
OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
"spec": {
|
||||||
|
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
"schedule": {
|
||||||
|
Type: "string",
|
||||||
|
},
|
||||||
|
"concurrencyPolicy": {
|
||||||
|
Enum: []apiextensionsv1beta1.JSON{
|
||||||
|
{Raw: []byte(`"Allow"`)},
|
||||||
|
{Raw: []byte(`"Forbid"`)},
|
||||||
|
{Raw: []byte(`"Replace"`)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"template": {
|
||||||
|
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
"type": {
|
||||||
|
Enum: []apiextensionsv1beta1.JSON{
|
||||||
|
{Raw: []byte(`"Java"`)},
|
||||||
|
{Raw: []byte(`"Scala"`)},
|
||||||
|
{Raw: []byte(`"Python"`)},
|
||||||
|
{Raw: []byte(`"R"`)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"mode": {
|
||||||
|
Enum: []apiextensionsv1beta1.JSON{
|
||||||
|
{Raw: []byte(`"cluster"`)},
|
||||||
|
{Raw: []byte(`"client"`)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"driver": {
|
||||||
|
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
"cores": {
|
||||||
|
Type: "number",
|
||||||
|
Minimum: float64Ptr(0),
|
||||||
|
ExclusiveMinimum: true,
|
||||||
|
},
|
||||||
|
"podName": {
|
||||||
|
Pattern: "[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"executor": {
|
||||||
|
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
"cores": {
|
||||||
|
Type: "integer",
|
||||||
|
Minimum: float64Ptr(1),
|
||||||
|
},
|
||||||
|
"instances": {
|
||||||
|
Type: "integer",
|
||||||
|
Minimum: float64Ptr(1),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"deps": {
|
||||||
|
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
|
||||||
|
"downloadTimeout": {
|
||||||
|
Type: "integer",
|
||||||
|
Minimum: float64Ptr(1),
|
||||||
|
},
|
||||||
|
"maxSimultaneousDownloads": {
|
||||||
|
Type: "integer",
|
||||||
|
Minimum: float64Ptr(1),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"restartPolicy": {
|
||||||
|
Enum: []apiextensionsv1beta1.JSON{
|
||||||
|
{Raw: []byte(`"Never"`)},
|
||||||
|
{Raw: []byte(`"OnFailure"`)},
|
||||||
|
{Raw: []byte(`"Always"`)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"maxSubmissionRetries": {
|
||||||
|
Type: "integer",
|
||||||
|
Minimum: float64Ptr(1),
|
||||||
|
},
|
||||||
|
"submissionRetryInterval": {
|
||||||
|
Type: "integer",
|
||||||
|
Minimum: float64Ptr(1),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func float64Ptr(f float64) *float64 {
|
||||||
|
return &f
|
||||||
|
}
|
||||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package crd
|
package sparkapplication
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
Loading…
Reference in New Issue