kubeflow
/
spark-operator
mirror of https://github.com/kubeflow/spark-operator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Miscellaneous Prometheus fixes

This commit is contained in:
Chaoran Yu 2018-09-19 21:40:45 -04:00 committed by Yinan Li
parent c43465e832
commit 2648ac2b2e
5 changed files with 295 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
Gopkg.lock generated
View File

@ -2,6 +2,7 @@
[[projects]]
digest = "1:9702dc153c9bb6ee7ee0587c248b7024700e89e4a7be284faaeeab9da32e1c6b"
name = "cloud.google.com/go"
packages = [
"compute/metadata",
@ -9,23 +10,62 @@
"internal",
"internal/optional",
"internal/version",
"storage"
"storage",
]
pruneopts = ""
revision = "767c40d6a2e058483c25fa193e963a22da17236d"
version = "v0.18.0"
[[projects]]
digest = "1:f1f464dded149767f5ceb579e73628139ee1a63d9022602109d439862b18f708"
name = "github.com/Azure/go-autorest"
packages = [
"autorest",
"autorest/adal",
"autorest/azure",
"autorest/date"
"autorest/date",
]
pruneopts = ""
revision = "fc3b03a2d2d1f43fad3007038bd16f044f870722"
version = "v9.10.0"
[[projects]]
branch = "master"
digest = "1:a01ad406efc79e2cdf827baa8836eb7a0ceb84bcec7f389a163076748076d4ab"
name = "github.com/GoogleCloudPlatform/spark-on-k8s-operator"
packages = [
"pkg/apis/sparkoperator.k8s.io",
"pkg/apis/sparkoperator.k8s.io/v1alpha1",
"pkg/apis/sparkoperator.k8s.io/v1beta1",
"pkg/client/clientset/versioned",
"pkg/client/clientset/versioned/fake",
"pkg/client/clientset/versioned/scheme",
"pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1alpha1",
"pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1alpha1/fake",
"pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1beta1",
"pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1beta1/fake",
"pkg/client/informers/externalversions",
"pkg/client/informers/externalversions/internalinterfaces",
"pkg/client/informers/externalversions/sparkoperator.k8s.io",
"pkg/client/informers/externalversions/sparkoperator.k8s.io/v1alpha1",
"pkg/client/informers/externalversions/sparkoperator.k8s.io/v1beta1",
"pkg/client/listers/sparkoperator.k8s.io/v1alpha1",
"pkg/client/listers/sparkoperator.k8s.io/v1beta1",
"pkg/config",
"pkg/controller/scheduledsparkapplication",
"pkg/controller/sparkapplication",
"pkg/crd",
"pkg/crd/scheduledsparkapplication",
"pkg/crd/sparkapplication",
"pkg/util",
"pkg/webhook",
"sparkctl/cmd",
]
pruneopts = ""
revision = "c43465e832dea23675f33a66698e680d900c5ddb"
[[projects]]
digest = "1:dbab841ea152563f54241d32eed59754b50fb46241f5b07b629f4bf2411b7474"
name = "github.com/aws/aws-sdk-go"
packages = [
"aws",
@ -59,78 +99,100 @@
"service/s3",
"service/s3/s3iface",
"service/s3/s3manager",
"service/sts"
"service/sts",
]
pruneopts = ""
revision = "92c96c4f5ad419f2a72401c2f606d51e010cc110"
version = "v1.14.6"
[[projects]]
branch = "master"
digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb"
name = "github.com/beorn7/perks"
packages = ["quantile"]
pruneopts = ""
revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
[[projects]]
digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = ""
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:6098222470fe0172157ce9bbef5d2200df4edde17ee649c5d6e48330e4afa4c6"
name = "github.com/dgrijalva/jwt-go"
packages = ["."]
pruneopts = ""
revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
version = "v3.2.0"
[[projects]]
branch = "master"
digest = "1:6c48291ff15f3c3b263ed6f7356acea45da69e7fca8d0d76d2c021a075fbd52a"
name = "github.com/docker/spdystream"
packages = [
".",
"spdy"
"spdy",
]
pruneopts = ""
revision = "bc6354cbbc295e925e4c611ffe90c1f287ee54db"
[[projects]]
digest = "1:dcefbadf4534c5ecac8573698fba6e6e601157bfa8f96aafe29df31ae582ef2a"
name = "github.com/evanphx/json-patch"
packages = ["."]
pruneopts = ""
revision = "afac545df32f2287a079e2dfb7ba2745a643747e"
version = "v3.0.0"
[[projects]]
digest = "1:b13707423743d41665fd23f0c36b2f37bb49c30e94adb813319c44188a51ba22"
name = "github.com/ghodss/yaml"
packages = ["."]
pruneopts = ""
revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
version = "v1.0.0"
[[projects]]
digest = "1:617b3e0f5989d4ff866a1820480990c65dfc9257eb080da749a45e2d76681b02"
name = "github.com/go-ini/ini"
packages = ["."]
pruneopts = ""
revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5"
version = "v1.37.0"
[[projects]]
digest = "1:0a3f6a0c68ab8f3d455f8892295503b179e571b7fefe47cc6c556405d1f83411"
name = "github.com/gogo/protobuf"
packages = [
"proto",
"sortkeys"
"sortkeys",
]
pruneopts = ""
revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:107b233e45174dbab5b1324201d092ea9448e58243ab9f039e4c0f332e121e3a"
name = "github.com/golang/glog"
packages = ["."]
pruneopts = ""
revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
[[projects]]
branch = "master"
digest = "1:1d8a57fce1f68298ce54967c0752a2ab54bf55dff261d245b8f3440a217700cb"
name = "github.com/golang/groupcache"
packages = ["lru"]
pruneopts = ""
revision = "24b0969c4cb722950103eed87108c8d291a8df00"
[[projects]]
digest = "1:f958a1c137db276e52f0b50efee41a1a389dcdded59a69711f3e872757dab34b"
name = "github.com/golang/protobuf"
packages = [
"proto",
@ -138,12 +200,14 @@
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp"
"ptypes/timestamp",
]
pruneopts = ""
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.1.0"
[[projects]]
digest = "1:3f95eb48889ea2863cd451b1e8066c725f64772fa06d69e675722dc5c95fd07f"
name = "github.com/google/go-cloud"
packages = [
"blob",
@ -151,35 +215,43 @@
"blob/gcsblob",
"blob/s3blob",
"gcp",
"wire"
"wire",
]
pruneopts = ""
revision = "354ed1d60b1bb1e190b483a8e8ce88053f83e904"
version = "v0.1.0"
[[projects]]
branch = "master"
digest = "1:754f77e9c839b24778a4b64422236d38515301d2baeb63113aa3edc42e6af692"
name = "github.com/google/gofuzz"
packages = ["."]
pruneopts = ""
revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
[[projects]]
digest = "1:e097a364f4e8d8d91b9b9eeafb992d3796a41fde3eb548c1a87eb9d9f60725cf"
name = "github.com/googleapis/gax-go"
packages = ["."]
pruneopts = ""
revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f"
version = "v2.0.0"
[[projects]]
digest = "1:16b2837c8b3cf045fa2cdc82af0cf78b19582701394484ae76b2c3bc3c99ad73"
name = "github.com/googleapis/gnostic"
packages = [
"OpenAPIv2",
"compiler",
"extensions"
"extensions",
]
pruneopts = ""
revision = "7c663266750e7d82587642f65e60bc4083f1f84e"
version = "v0.2.0"
[[projects]]
branch = "master"
digest = "1:4301e05db7d213e68f2ea25e53809136218c7efb2ebfff4efed949b91adc7a2a"
name = "github.com/gophercloud/gophercloud"
packages = [
".",
@ -188,144 +260,186 @@
"openstack/identity/v2/tokens",
"openstack/identity/v3/tokens",
"openstack/utils",
"pagination"
"pagination",
]
pruneopts = ""
revision = "6cd928389e7eefe1715e188b97fdd07b09a55e62"
[[projects]]
branch = "master"
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
name = "github.com/hashicorp/golang-lru"
packages = [
".",
"simplelru"
"simplelru",
]
pruneopts = ""
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
[[projects]]
branch = "master"
digest = "1:f81c8d7354cc0c6340f2f7a48724ee6c2b3db3e918ecd441c985b4d2d97dd3e7"
name = "github.com/howeyc/gopass"
packages = ["."]
pruneopts = ""
revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
[[projects]]
digest = "1:302c6eb8e669c997bec516a138b8fc496018faa1ece4c13e445a2749fbe079bb"
name = "github.com/imdario/mergo"
packages = ["."]
pruneopts = ""
revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58"
version = "v0.3.5"
[[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
pruneopts = ""
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e"
name = "github.com/jmespath/go-jmespath"
packages = ["."]
pruneopts = ""
revision = "0b12d6b5"
[[projects]]
digest = "1:9eab2325abbed0ebcee9d44bb3660a69d5d10e42d5ac4a0e77f7a6ea22bfce88"
name = "github.com/json-iterator/go"
packages = ["."]
pruneopts = ""
revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4"
version = "1.1.3"
[[projects]]
digest = "1:81e673df85e765593a863f67cba4544cf40e8919590f04d67664940786c2b61a"
name = "github.com/mattn/go-runewidth"
packages = ["."]
pruneopts = ""
revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
version = "v0.0.2"
[[projects]]
digest = "1:63722a4b1e1717be7b98fc686e0b30d5e7f734b9e93d7dee86293b6deab7ea28"
name = "github.com/matttproud/golang_protobuf_extensions"
packages = ["pbutil"]
pruneopts = ""
revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
version = "v1.0.1"
[[projects]]
digest = "1:0c0ff2a89c1bb0d01887e1dac043ad7efbf3ec77482ef058ac423d13497e16fd"
name = "github.com/modern-go/concurrent"
packages = ["."]
pruneopts = ""
revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
version = "1.0.3"
[[projects]]
digest = "1:420f9231f816eeca3ff5aab070caac3ed7f27e4d37ded96ce9de3d7a7a2e31ad"
name = "github.com/modern-go/reflect2"
packages = ["."]
pruneopts = ""
revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f"
version = "1.0.0"
[[projects]]
branch = "master"
digest = "1:816e800c7b5689e114c0dc6acf494553d1e7a52575764c5d6de67fb32bd8bbb2"
name = "github.com/olekukonko/tablewriter"
packages = ["."]
pruneopts = ""
revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279"
[[projects]]
digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = ""
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:3148d6840726c78a90ca3e712870c43eebfd723af101a4917adab87bc3da3441"
name = "github.com/prometheus/client_golang"
packages = [
"prometheus",
"prometheus/promhttp"
"prometheus/promhttp",
]
pruneopts = ""
revision = "ee1c9d7e23df7f011bdf6f12a5c9e7f0ae10a1fe"
[[projects]]
branch = "master"
digest = "1:185cf55b1f44a1bf243558901c3f06efa5c64ba62cfdcbb1bf7bbe8c3fb68561"
name = "github.com/prometheus/client_model"
packages = ["go"]
pruneopts = ""
revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
[[projects]]
branch = "master"
digest = "1:bfbc121ef802d245ef67421cff206615357d9202337a3d492b8f668906b485a8"
name = "github.com/prometheus/common"
packages = [
"expfmt",
"internal/bitbucket.org/ww/goautoneg",
"model"
"model",
]
pruneopts = ""
revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
[[projects]]
branch = "master"
digest = "1:b694a6bdecdace488f507cff872b30f6f490fdaf988abd74d87ea56406b23b6e"
name = "github.com/prometheus/procfs"
packages = [
".",
"internal/util",
"nfs",
"xfs"
"xfs",
]
pruneopts = ""
revision = "ae68e2d4c00fed4943b5f6698d504a5fe083da8a"
[[projects]]
digest = "1:4fc12a4611ba6e3db7dd567580579cb23fbe59ac77e7233dc8defa7cb0c62001"
name = "github.com/robfig/cron"
packages = ["."]
pruneopts = ""
revision = "df38d32658d8788cd446ba74db4bb5375c4b0cb3"
[[projects]]
digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
name = "github.com/spf13/cobra"
packages = ["."]
pruneopts = ""
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
version = "v0.0.3"
[[projects]]
digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
name = "github.com/spf13/pflag"
packages = ["."]
pruneopts = ""
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
version = "v1.0.1"
[[projects]]
digest = "1:c587772fb8ad29ad4db67575dad25ba17a51f072ff18a22b4f0257a4d9c24f75"
name = "github.com/stretchr/testify"
packages = ["assert"]
pruneopts = ""
revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
version = "v1.2.2"
[[projects]]
digest = "1:7ae9ab20a379344cd296d7cd041180d547ba5489b8fd6e8919e0de24a94991b2"
name = "go.opencensus.io"
packages = [
".",
@ -340,19 +454,23 @@
"tag",
"trace",
"trace/internal",
"trace/propagation"
"trace/propagation",
]
pruneopts = ""
revision = "6edeb78af2d9e4f169abb223feaef35da2e45d06"
version = "v0.13.0"
[[projects]]
branch = "master"
digest = "1:9790f02de4149564057296b2698e3632acb48ec5471eb5cc89bf93364ee5f2ea"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
pruneopts = ""
revision = "a8fb68e7206f8c78be19b432c58eb52a6aa34462"
[[projects]]
branch = "master"
digest = "1:5dc6753986b9eeba4abdf05dedc5ba06bb52dad43cc8aad35ffb42bb7adfa68f"
name = "golang.org/x/net"
packages = [
"context",
@ -362,32 +480,38 @@
"http2/hpack",
"idna",
"internal/timeseries",
"trace"
"trace",
]
pruneopts = ""
revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"
[[projects]]
branch = "master"
digest = "1:823e7b6793b3f80b5d01da97211790dc89601937e4b70825fdcb5637ac60f04f"
name = "golang.org/x/oauth2"
packages = [
".",
"google",
"internal",
"jws",
"jwt"
"jwt",
]
pruneopts = ""
revision = "1e0a3fa8ba9a5c9eb35c271780101fdaf1b205d7"
[[projects]]
branch = "master"
digest = "1:14ef7df419735d494ee96b2cbc653f4209bd6634125bb66a63c4bcf5f5ab6bf3"
name = "golang.org/x/sys"
packages = [
"unix",
"windows"
"windows",
]
pruneopts = ""
revision = "8883426083c04a2627e6e59d84d5f6fb63d16c91"
[[projects]]
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
name = "golang.org/x/text"
packages = [
"collate",
@ -403,29 +527,35 @@
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable"
"unicode/rangetable",
]
pruneopts = ""
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
branch = "master"
digest = "1:55a681cb66f28755765fa5fa5104cbd8dc85c55c02d206f9f89566451e3fe1aa"
name = "golang.org/x/time"
packages = ["rate"]
pruneopts = ""
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
[[projects]]
branch = "master"
digest = "1:b3e50d2cb7f622d4c647006c4ea5a3245383f3050c31bc246df8c31034f129fe"
name = "golang.org/x/tools"
packages = [
"go/ast/astutil",
"imports",
"internal/fastwalk"
"internal/fastwalk",
]
pruneopts = ""
revision = "c6269430a55499efa2e293ae66926175dc3af30e"
[[projects]]
branch = "master"
digest = "1:7d15746ff4df12481c89fd953a28122fa75368fb1fb1bb1fed918a78647b3c3a"
name = "google.golang.org/api"
packages = [
"gensupport",
@ -436,11 +566,13 @@
"iterator",
"option",
"storage/v1",
"transport/http"
"transport/http",
]
pruneopts = ""
revision = "2eea9ba0a3d94f6ab46508083e299a00bbbc65f6"
[[projects]]
digest = "1:934fb8966f303ede63aa405e2c8d7f0a427a05ea8df335dfdc1833dd4d40756f"
name = "google.golang.org/appengine"
packages = [
".",
@ -452,22 +584,26 @@
"internal/modules",
"internal/remote_api",
"internal/urlfetch",
"urlfetch"
"urlfetch",
]
pruneopts = ""
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:d88c2eb6750028f5707971ed1340a02407d5f0967af85bf6ac70c031533f6c15"
name = "google.golang.org/genproto"
packages = [
"googleapis/api/annotations",
"googleapis/iam/v1",
"googleapis/rpc/status"
"googleapis/rpc/status",
]
pruneopts = ""
revision = "32ee49c4dd805befd833990acba36cb75042378c"
[[projects]]
digest = "1:dda79c5192c1c59f3f60bbc109a6c710f37e3ebef4d1e7527f586e11ec9ba2af"
name = "google.golang.org/grpc"
packages = [
".",
@ -493,25 +629,31 @@
"stats",
"status",
"tap",
"transport"
"transport",
]
pruneopts = ""
revision = "7a6a684ca69eb4cae85ad0a484f2e531598c047b"
version = "v1.12.2"
[[projects]]
digest = "1:75fb3fcfc73a8c723efde7777b40e8e8ff9babf30d8c56160d01beffea8a95a6"
name = "gopkg.in/inf.v0"
packages = ["."]
pruneopts = ""
revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
version = "v0.9.1"
[[projects]]
digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = ""
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1"
[[projects]]
branch = "release-1.10"
digest = "1:5beb32094452970c0d73a2bdacd79aa9cfaa4947a774d521c1bed4b4c2705f15"
name = "k8s.io/api"
packages = [
"admission/v1beta1",
@ -542,12 +684,14 @@
"settings/v1alpha1",
"storage/v1",
"storage/v1alpha1",
"storage/v1beta1"
"storage/v1beta1",
]
pruneopts = ""
revision = "8b7507fac302640dd5f1efbf9643199952cc58db"
[[projects]]
branch = "release-1.10"
digest = "1:66d1421ecff35bc48ee0b11a3f891f3af6f775ed6bb1d3e0deeaba221bf42490"
name = "k8s.io/apiextensions-apiserver"
packages = [
"pkg/apis/apiextensions",
@ -556,12 +700,14 @@
"pkg/client/clientset/clientset/fake",
"pkg/client/clientset/clientset/scheme",
"pkg/client/clientset/clientset/typed/apiextensions/v1beta1",
"pkg/client/clientset/clientset/typed/apiextensions/v1beta1/fake"
"pkg/client/clientset/clientset/typed/apiextensions/v1beta1/fake",
]
pruneopts = ""
revision = "8e7f43002fec5394a8d96ebca781aa9d4b37aaef"
[[projects]]
branch = "release-1.10"
digest = "1:ddf1cbe17938e59f1f5d7caa57c03382a450fdc5c91e5852ff89802c8b27b459"
name = "k8s.io/apimachinery"
packages = [
"pkg/api/errors",
@ -608,12 +754,14 @@
"pkg/watch",
"third_party/forked/golang/json",
"third_party/forked/golang/netutil",
"third_party/forked/golang/reflect"
"third_party/forked/golang/reflect",
]
pruneopts = ""
revision = "17529ec7eadb8de8e7dc835201455f53571f655a"
[[projects]]
branch = "release-7.0"
digest = "1:3a45889089f89cc371fb45b3f8a478248b755e4af17a8cf592e49bdf3481a0b3"
name = "k8s.io/client-go"
packages = [
"discovery",
@ -710,12 +858,14 @@
"util/integer",
"util/jsonpath",
"util/retry",
"util/workqueue"
"util/workqueue",
]
pruneopts = ""
revision = "26a26f55b28aa1b338fbaf6fbbe0bcd76aed05e0"
[[projects]]
branch = "master"
digest = "1:5f8a121f3d6f52aa450f560b20096b9d622fead09b0d318d31343af7bc8dbf44"
name = "k8s.io/code-generator"
packages = [
"cmd/client-gen",
@ -730,12 +880,14 @@
"cmd/deepcopy-gen/args",
"cmd/defaulter-gen",
"cmd/defaulter-gen/args",
"pkg/util"
"pkg/util",
]
pruneopts = ""
revision = "4c99649af8fee16989100e4cc3e6a75143530b28"
[[projects]]
branch = "master"
digest = "1:0a35c959fd117efeda39643d60ce6a0530cd6649ad9ebaf1e56c02baed367a80"
name = "k8s.io/gengo"
packages = [
"args",
@ -745,25 +897,117 @@
"generator",
"namer",
"parser",
"types"
"types",
]
pruneopts = ""
revision = "dcbe4570f0cf6efbc583a5321c8f9390f71a544d"
[[projects]]
branch = "master"
digest = "1:324305490fa0113cac244a48e648bf8a566f0a6e5e25c498e8c033a0b4e23c96"
name = "k8s.io/kube-openapi"
packages = ["pkg/util/proto"]
pruneopts = ""
revision = "bf40560368791a7dddfeea9b3cfcf89b34139f44"
[[projects]]
digest = "1:13614139aef760b502fe057a8d34e9c4257d1581fb85498b0053267596e0514c"
name = "k8s.io/kubernetes"
packages = ["pkg/util/interrupt"]
pruneopts = ""
revision = "bb9ffb1654d4a729bb4cec18ff088eacc153c239"
version = "v1.11.2"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "a6b3389ce59992ab7fa5c82b0dc47146142d0df41c7295526c39bacc4de07289"
input-imports = [
"cloud.google.com/go/storage",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1alpha1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1beta1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned/fake",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned/scheme",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1alpha1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1alpha1/fake",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1beta1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/clientset/versioned/typed/sparkoperator.k8s.io/v1beta1/fake",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions/internalinterfaces",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions/sparkoperator.k8s.io",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions/sparkoperator.k8s.io/v1alpha1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions/sparkoperator.k8s.io/v1beta1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/listers/sparkoperator.k8s.io/v1alpha1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/listers/sparkoperator.k8s.io/v1beta1",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/config",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/controller/scheduledsparkapplication",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/controller/sparkapplication",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/crd",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/crd/scheduledsparkapplication",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/crd/sparkapplication",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/util",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/webhook",
"github.com/GoogleCloudPlatform/spark-on-k8s-operator/sparkctl/cmd",
"github.com/aws/aws-sdk-go/aws",
"github.com/aws/aws-sdk-go/aws/session",
"github.com/aws/aws-sdk-go/service/s3",
"github.com/evanphx/json-patch",
"github.com/golang/glog",
"github.com/google/go-cloud/blob",
"github.com/google/go-cloud/blob/gcsblob",
"github.com/google/go-cloud/blob/s3blob",
"github.com/google/go-cloud/gcp",
"github.com/olekukonko/tablewriter",
"github.com/prometheus/client_golang/prometheus",
"github.com/prometheus/client_golang/prometheus/promhttp",
"github.com/prometheus/client_model/go",
"github.com/robfig/cron",
"github.com/spf13/cobra",
"github.com/stretchr/testify/assert",
"golang.org/x/net/context",
"k8s.io/api/admission/v1beta1",
"k8s.io/api/admissionregistration/v1beta1",
"k8s.io/api/core/v1",
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1",
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset",
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake",
"k8s.io/apimachinery/pkg/api/errors",
"k8s.io/apimachinery/pkg/apis/meta/v1",
"k8s.io/apimachinery/pkg/labels",
"k8s.io/apimachinery/pkg/runtime",
"k8s.io/apimachinery/pkg/runtime/schema",
"k8s.io/apimachinery/pkg/runtime/serializer",
"k8s.io/apimachinery/pkg/types",
"k8s.io/apimachinery/pkg/util/clock",
"k8s.io/apimachinery/pkg/util/duration",
"k8s.io/apimachinery/pkg/util/errors",
"k8s.io/apimachinery/pkg/util/runtime",
"k8s.io/apimachinery/pkg/util/wait",
"k8s.io/apimachinery/pkg/util/yaml",
"k8s.io/apimachinery/pkg/watch",
"k8s.io/client-go/discovery",
"k8s.io/client-go/discovery/fake",
"k8s.io/client-go/kubernetes",
"k8s.io/client-go/kubernetes/fake",
"k8s.io/client-go/kubernetes/scheme",
"k8s.io/client-go/kubernetes/typed/core/v1",
"k8s.io/client-go/plugin/pkg/client/auth",
"k8s.io/client-go/plugin/pkg/client/auth/gcp",
"k8s.io/client-go/rest",
"k8s.io/client-go/testing",
"k8s.io/client-go/tools/cache",
"k8s.io/client-go/tools/clientcmd",
"k8s.io/client-go/tools/portforward",
"k8s.io/client-go/tools/record",
"k8s.io/client-go/transport/spdy",
"k8s.io/client-go/util/flowcontrol",
"k8s.io/client-go/util/retry",
"k8s.io/client-go/util/workqueue",
"k8s.io/code-generator/cmd/client-gen",
"k8s.io/code-generator/cmd/deepcopy-gen",
"k8s.io/code-generator/cmd/defaulter-gen",
"k8s.io/kubernetes/pkg/util/interrupt",
]
solver-name = "gps-cdcl"
solver-version = 1

27
docs/user-guide.md
View File

@ -1,12 +1,7 @@
# User Guide
For a quick introduction on how to build and install the Kubernetes Operator for Apache Spark, and how to run some example applications,
please refer to the [Quick Start Guide](quick-start-guide.md). For a complete reference of the API definition of the
`SparkApplication` and `ScheduledSparkApplication` custom resources, please refer to the [API Specification](api.md).
The Kubernetes Operator for Apache Spark ships with a command-line tool called `sparkctl` that offers additional features beyond what `kubectl`
is able to do. Documentation on `sparkctl` can be found in [README](../sparkctl/README.md). If you are running the Spark
Operator on Google Kubernetes Engine and want to use Google Cloud Storage (GCS) and/or BigQuery for reading/writing data,
also refer to the [GCP guide](gcp.md). The Kubernetes Operator for Apache Spark will simply be referred to as the operator for the rest of this guide.
For a quick introduction on how to build and install the Kubernetes Operator for Apache Spark, and how to run some example applications, please refer to the [Quick Start Guide](quick-start-guide.md). For a complete reference of the API definition of the `SparkApplication` and `ScheduledSparkApplication` custom resources, please refer to the [API Specification](api.md).
The Kubernetes Operator for Apache Spark ships with a command-line tool called `sparkctl` that offers additional features beyond what `kubectl` is able to do. Documentation on `sparkctl` can be found in [README](../sparkctl/README.md). If you are running the Spark Operator on Google Kubernetes Engine and want to use Google Cloud Storage (GCS) and/or BigQuery for reading/writing data, also refer to the [GCP guide](gcp.md). The Kubernetes Operator for Apache Spark will simply be referred to as the operator for the rest of this guide.
## Table of Contents
* [Using a SparkApplication](#using-a-sparkapplication)
@ -103,7 +98,7 @@ spec:
files:
- gs://spark-data/data-file-1.txt
- gs://spark-data/data-file-2.txt
```
```
### Specifying Spark Configuration
@ -218,7 +213,7 @@ spec:
- name: gcp-svc-account
path: /mnt/secrets
secretType: GCPServiceAccount
```
```
The type of a Secret as specified by the `secretType` field is a hint to the operator on what extra configuration
it needs to take care of for the specific type of Secrets. For example, if a Secret is of type **`GCPServiceAccount`**, the
@ -326,7 +321,7 @@ spec:
SECRET_PASSWORD:
name: mysecret
key: password
```
```
### Using Image Pull Secrets
@ -387,22 +382,22 @@ spec:
pyfiles:
- local:///opt/spark/examples/src/main/python/py_container_checks.py
- gs://spark-data/python-dep.zip
```
```
In order to use the dependencies that are hosted remotely, the following PySpark code
can be used in Spark 2.4.
```
python_dep_file_path = SparkFiles.get("python-dep.zip")
spark.sparkContext.addPyFile(dep_file_path)
```
```
Note that Python binding for PySpark will be available in Apache Spark 2.4,
and currently requires building a custom Docker image from the Spark master branch.
### Monitoring
The operator supports using the Spark metric system to expose metrics to a varity of sinks. Particularly, it is able to automatically configure the metric system to expose metrics to [Prometheus](https://prometheus.io/). Specifically, the field `.spec.monitoring` specifies how application monitoring is handled and particularly how metrics are to be reported. The metric system is configured through the configuration file `metrics.properties`, which gets its content from the field `.spec.monitoring.metricsProperties`. The content of [metrics.properties](../spark-docker/conf/metrics.properties) will be used by default if `.spec.monitoring.metricsProperties` is not specified. You can choose to enable or disable reporting driver and executor metrics using the fields `.spec.monitoring.exposeDriverMetrics` and `.spec.monitoring.exposeExecutorMetrics`, respectively.
The operator supports using the Spark metric system to expose metrics to a variety of sinks. Particularly, it is able to automatically configure the metric system to expose metrics to [Prometheus](https://prometheus.io/). Specifically, the field `.spec.monitoring` specifies how application monitoring is handled and particularly how metrics are to be reported. The metric system is configured through the configuration file `metrics.properties`, which gets its content from the field `.spec.monitoring.metricsProperties`. The content of [metrics.properties](../spark-docker/conf/metrics.properties) will be used by default if `.spec.monitoring.metricsProperties` is not specified. You can choose to enable or disable reporting driver and executor metrics using the fields `.spec.monitoring.exposeDriverMetrics` and `.spec.monitoring.exposeExecutorMetrics`, respectively.
Further, the field `.spec.monitoring.prometheus` specifies how metrics are exposed to Prometheus using the [Prometheus JMX exporter](https://github.com/prometheus/jmx_exporter). When `.spec.monitoring.prometheus` is specified, the operator automatically configures the JMX exporter to run as a Java agent. The only required field of `.spec.monitoring.prometheus` is `jmxExporterJar`, which specified the path to the Prometheus JMX exporter Java agent jar in the container. If you use the image `gcr.io/spark-operator/spark:v2.3.1-gcs-prometheus`, the jar is located at `/prometheus/jmx_prometheus_javaagent-0.3.1.jar`. The field `.spec.monitoring.prometheus.port` specifies the port the JMX exporter Java agent binds to and defaults to `8090` if not specified. The field `.spec.monitoring.prometheus.configuration` specifies the content of the configuration to be used with the JMX exporter. The content of [prometheus.yaml](../spark-docker/conf/prometheus.yaml) will be used by default if `.spec.monitoring.prometheus.configuration` is not specified.
@ -419,7 +414,7 @@ spec:
jmxExporterJar: "/var/spark-data/spark-jars/jmx_prometheus_javaagent-0.3.1.jar"
```
The operator automatically adds the annotations such as `prometheus.io/scrape=true` on the driver and/or executor pods so the metrics exposed on the pods can be scraped by the Prometheus server in the same cluster.
The operator automatically adds the annotations such as `prometheus.io/scrape=true` on the driver and/or executor pods (depending on the values of `.spec.monitoring.exposeDriverMetrics` and `.spec.monitoring.exposeExecutorMetrics`) so the metrics exposed on the pods can be scraped by the Prometheus server in the same cluster.
## Working with SparkApplications
@ -537,7 +532,7 @@ schedule and concurrency policy of a `ScheduledSparkApplication`. For example, a
be used with a `ScheduledSparkApplication`. In most cases, a restart policy of `OnFailure` may not be a good choice as
the next run usually picks up where the previous run left anyway. For these reasons, it's often the right choice to use
a restart policy of `Never` as the example above shows.
## Customizing the Operator
To customize the operator, you can follow the steps below:

6
examples/spark-pi-prometheus.yaml
View File

@ -22,7 +22,7 @@ metadata:
spec:
type: Scala
mode: cluster
image: "gcr.io/spark-operator/spark:v2.3.1"
image: "gcr.io/spark-operator/spark:v2.3.1-gcs-prometheus"
imagePullPolicy: Always
mainClass: org.apache.spark.examples.SparkPi
mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar"
@ -44,6 +44,8 @@ spec:
version: 2.3.1
monitoring:
exposeDriverMetrics: true
exposeExecutorMetrics: true
prometheus:
jmxExporterJar: "/var/spark-data/spark-jars/jmx_prometheus_javaagent-0.3.1.jar"
jmxExporterJar: "/var/spark-data/spark-jars/jmx_prometheus_javaagent-0.3.1.jar"
port: 8090
restartPolicy: Never

9
spark-docker/Dockerfile
View File

@ -14,7 +14,8 @@
# limitations under the License.
#
FROM gcr.io/spark-operator/spark:v2.3.1
ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.3.1
FROM ${SPARK_IMAGE}
# Setup dependencies for Google Cloud Storage access.
RUN rm $SPARK_HOME/jars/guava-14.0.1.jar
@ -23,10 +24,10 @@ ADD http://central.maven.org/maven2/com/google/guava/guava/23.0/guava-23.0.jar $
ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-latest-hadoop2.jar $SPARK_HOME/jars
# Setup for the Prometheus JMX exporter.
RUN mkdir /prometheus
RUN mkdir -p /etc/metrics/conf
# Add the Prometheus JMX exporter Java agent jar for exposing metrics sent to the JmxSink to Prometheus.
ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar /prometheus/
COPY conf/metrics.properties /prometheus/
COPY conf/prometheus.yaml /prometheus/
COPY conf/metrics.properties /etc/metrics/conf
COPY conf/prometheus.yaml /etc/metrics/conf
ENTRYPOINT ["/opt/entrypoint.sh"]

2
spark-docker/conf/metrics.properties
View File

@ -16,4 +16,4 @@
*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink
driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource
executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource
executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource