kubeflow
/
spark-operator
mirror of https://github.com/kubeflow/spark-operator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updated CRD yamls and API docs (#749)

This commit is contained in:
Yinan Li 2019-12-19 16:21:09 -08:00 committed by GitHub
parent 52dc9a6412
commit bbc3e71442
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2230 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
docs/api-docs.md
View File

@ -279,7 +279,8 @@ string
</td>
<td>
<em>(Optional)</em>
<p>InitContainerImage is the image of the init-container to use. Overrides Spec.Image if set.</p>
<p>InitContainerImage is the image of the init-container to use. Overrides Spec.Image if set.
Deprecated.</p>
</td>
</tr>
<tr>
@ -764,7 +765,8 @@ string
</td>
<td>
<em>(Optional)</em>
<p>JarsDownloadDir is the location to download jars to in the driver and executors.</p>
<p>JarsDownloadDir is the location to download jars to in the driver and executors.
Deprecated.</p>
</td>
</tr>
<tr>
@ -776,7 +778,8 @@ string
</td>
<td>
<em>(Optional)</em>
<p>FilesDownloadDir is the location to download files to in the driver and executors.</p>
<p>FilesDownloadDir is the location to download files to in the driver and executors.
Deprecated.</p>
</td>
</tr>
<tr>
@ -789,7 +792,8 @@ int32
<td>
<em>(Optional)</em>
<p>DownloadTimeout specifies the timeout in seconds before aborting the attempt to download
and unpack dependencies from remote locations into the driver and executor pods.</p>
and unpack dependencies from remote locations into the driver and executor pods.
Deprecated.</p>
</td>
</tr>
<tr>
@ -802,7 +806,8 @@ int32
<td>
<em>(Optional)</em>
<p>MaxSimultaneousDownloads specifies the maximum number of remote dependencies to download
simultaneously in a driver or executor pod.</p>
simultaneously in a driver or executor pod.
Deprecated.</p>
</td>
</tr>
</tbody>
@ -965,6 +970,7 @@ string
</em>
</td>
<td>
<em>(Optional)</em>
<p>ServiceAccount is the name of the Kubernetes service account used by the driver pod
when requesting executor pods from the API server.</p>
</td>
@ -977,6 +983,7 @@ string
</em>
</td>
<td>
<em>(Optional)</em>
<p>JavaOptions is a string of extra JVM options to pass to the driver. For instance,
GC settings or other logging.</p>
</td>
@ -1313,6 +1320,7 @@ string
</em>
</td>
<td>
<em>(Optional)</em>
<p>ConfigFile is the path to the custom Prometheus configuration file provided in the Spark image.
ConfigFile takes precedence over Configuration, which is shown below.</p>
</td>
@ -1361,6 +1369,7 @@ RestartPolicyType
</em>
</td>
<td>
<p>Type specifies the RestartPolicyType.</p>
</td>
</tr>
<tr>
@ -1371,7 +1380,8 @@ int32
</em>
</td>
<td>
<p>FailureRetries are the number of times to retry a failed application before giving up in a particular case.
<em>(Optional)</em>
<p>OnSubmissionFailureRetries is the number of times to retry submitting an application before giving up.
This is best effort and actual retry attempts can be &gt;= the value specified due to caching.
These are required if RestartPolicy is OnFailure.</p>
</td>
@ -1384,6 +1394,8 @@ int32
</em>
</td>
<td>
<em>(Optional)</em>
<p>OnFailureRetries the number of times to retry running an application before giving up.</p>
</td>
</tr>
<tr>
@ -1394,7 +1406,9 @@ int64
</em>
</td>
<td>
<p>Interval to wait between successive retries of a failed application.</p>
<em>(Optional)</em>
<p>OnSubmissionFailureRetryInterval is the interval between retries on failed submissions.
Interval to wait between successive retries of a failed application.</p>
</td>
</tr>
<tr>
@ -1405,6 +1419,8 @@ int64
</em>
</td>
<td>
<em>(Optional)</em>
<p>OnFailureRetryInterval is the interval between retries on failed runs.</p>
</td>
</tr>
</tbody>
@ -1758,7 +1774,8 @@ string
</td>
<td>
<em>(Optional)</em>
<p>InitContainerImage is the image of the init-container to use. Overrides Spec.Image if set.</p>
<p>InitContainerImage is the image of the init-container to use. Overrides Spec.Image if set.
Deprecated.</p>
</td>
</tr>
<tr>
@ -2349,7 +2366,7 @@ map[string]string
<td>
<em>(Optional)</em>
<p>EnvVars carries the environment variables to add to the pod.
DEPRECATED.</p>
Deprecated.</p>
</td>
</tr>
<tr>
@ -2364,7 +2381,7 @@ map[string]github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkop
<td>
<em>(Optional)</em>
<p>EnvSecretKeyRefs holds a mapping from environment variable names to SecretKeyRefs.
DEPRECATED.</p>
Deprecated.</p>
</td>
</tr>
<tr>
@ -2531,5 +2548,5 @@ Kubernetes core/v1.PodDNSConfig
<hr/>
<p><em>
Generated with <code>gen-crd-api-reference-docs</code>
on git commit <code>1b96b7b</code>.
on git commit <code>76e05fe</code>.
</em></p>

1094
manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
View File

File diff suppressed because it is too large Load Diff

1121
manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
View File

File diff suppressed because it is too large Load Diff

33
pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
View File

@ -45,21 +45,31 @@ const (
// RestartPolicy is the policy of if and in which conditions the controller should restart a terminated application.
// This completely defines actions to be taken on any kind of Failures during an application run.
type RestartPolicy struct {
// Type specifies the RestartPolicyType.
// +kubebuilder:validation:Enum={Never,Always,OnFailure}
Type RestartPolicyType `json:"type,omitempty"`
// FailureRetries are the number of times to retry a failed application before giving up in a particular case.
// OnSubmissionFailureRetries is the number of times to retry submitting an application before giving up.
// This is best effort and actual retry attempts can be >= the value specified due to caching.
// These are required if RestartPolicy is OnFailure.
// +kubebuilder:validation:Minimum=0
// +optional
OnSubmissionFailureRetries *int32 `json:"onSubmissionFailureRetries,omitempty"`
// OnFailureRetries the number of times to retry running an application before giving up.
// +kubebuilder:validation:Minimum=0
// +optional
OnFailureRetries *int32 `json:"onFailureRetries,omitempty"`
// OnSubmissionFailureRetryInterval is the interval between retries on failed submissions.
// Interval to wait between successive retries of a failed application.
// +kubebuilder:validation:Minimum=1
// +optional
OnSubmissionFailureRetryInterval *int64 `json:"onSubmissionFailureRetryInterval,omitempty"`
// OnFailureRetryInterval is the interval between retries on failed runs.
// +kubebuilder:validation:Minimum=1
// +optional
OnFailureRetryInterval *int64 `json:"onFailureRetryInterval,omitempty"`
}
@ -180,8 +190,8 @@ type SparkApplicationSpec struct {
// +optional
Image *string `json:"image,omitempty"`
// InitContainerImage is the image of the init-container to use. Overrides Spec.Image if set.
// Deprecated.
// +optional
// +deprecated
InitContainerImage *string `json:"initContainerImage,omitempty"`
// ImagePullPolicy is the image pull policy for the driver, executor, and init-container.
// +optional
@ -358,23 +368,23 @@ type Dependencies struct {
// +optional
PyFiles []string `json:"pyFiles,omitempty"`
// JarsDownloadDir is the location to download jars to in the driver and executors.
// Deprecated.
// +optional
// +deprecated
JarsDownloadDir *string `json:"jarsDownloadDir,omitempty"`
// FilesDownloadDir is the location to download files to in the driver and executors.
// Deprecated.
// +optional
// +deprecated
FilesDownloadDir *string `json:"filesDownloadDir,omitempty"`
// DownloadTimeout specifies the timeout in seconds before aborting the attempt to download
// and unpack dependencies from remote locations into the driver and executor pods.
// Deprecated.
// +optional
// +deprecated
// +kubebuilder:validation:Minimum=1
DownloadTimeout *int32 `json:"downloadTimeout,omitempty"`
// MaxSimultaneousDownloads specifies the maximum number of remote dependencies to download
// simultaneously in a driver or executor pod.
// Deprecated.
// +optional
// +deprecated
// +kubebuilder:validation:Minimum=1
MaxSimultaneousDownloads *int32 `json:"maxSimultaneousDownloads,omitempty"`
}
@ -411,12 +421,12 @@ type SparkPodSpec struct {
// +optional
Env []apiv1.EnvVar `json:"env,omitempty"`
// EnvVars carries the environment variables to add to the pod.
// Deprecated.
// +optional
// DEPRECATED.
EnvVars map[string]string `json:"envVars,omitempty"`
// EnvSecretKeyRefs holds a mapping from environment variable names to SecretKeyRefs.
// Deprecated.
// +optional
// DEPRECATED.
EnvSecretKeyRefs map[string]NameKey `json:"envSecretKeyRefs,omitempty"`
// Labels are the Kubernetes labels to be added to the pod.
// +optional
@ -473,9 +483,11 @@ type DriverSpec struct {
CoreRequest *string `json:"coreRequest,omitempty"`
// ServiceAccount is the name of the Kubernetes service account used by the driver pod
// when requesting executor pods from the API server.
// +optional
ServiceAccount *string `json:"serviceAccount,omitempty"`
// JavaOptions is a string of extra JVM options to pass to the driver. For instance,
// GC settings or other logging.
// +optional
JavaOptions *string `json:"javaOptions,omitempty"`
}
@ -567,18 +579,19 @@ type PrometheusSpec struct {
// JmxExporterJar is the path to the Prometheus JMX exporter jar in the container.
JmxExporterJar string `json:"jmxExporterJar"`
// Port is the port of the HTTP server run by the Prometheus JMX exporter.
// +optional
// If not specified, 8090 will be used as the default.
// +kubebuilder:validation:Minimum=1024
// +kubebuilder:validation:Maximum=49151
// +optional
Port *int32 `json:"port,omitempty"`
// ConfigFile is the path to the custom Prometheus configuration file provided in the Spark image.
// ConfigFile takes precedence over Configuration, which is shown below.
// +optional
ConfigFile *string `json:"configFile,omitempty"`
// Configuration is the content of the Prometheus configuration needed by the Prometheus JMX exporter.
// +optional
// If not specified, the content in spark-docker/conf/prometheus.yaml will be used.
// Configuration has no effect if ConfigFile is set.
// +optional
Configuration *string `json:"configuration,omitempty"`
}