spark Constants 源码

spark Constants 代码

文件路径：/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.spark.deploy.k8s

private[spark] object Constants {

  // Labels
  val SPARK_VERSION_LABEL = "spark-version"
  val SPARK_APP_ID_LABEL = "spark-app-selector"
  val SPARK_APP_NAME_LABEL = "spark-app-name"
  val SPARK_EXECUTOR_ID_LABEL = "spark-exec-id"
  val SPARK_RESOURCE_PROFILE_ID_LABEL = "spark-exec-resourceprofile-id"
  val SPARK_ROLE_LABEL = "spark-role"
  val SPARK_POD_DRIVER_ROLE = "driver"
  val SPARK_POD_EXECUTOR_ROLE = "executor"
  val SPARK_EXECUTOR_INACTIVE_LABEL = "spark-exec-inactive"

  // Credentials secrets
  val DRIVER_CREDENTIALS_SECRETS_BASE_DIR =
    "/mnt/secrets/spark-kubernetes-credentials"
  val DRIVER_CREDENTIALS_CA_CERT_SECRET_NAME = "ca-cert"
  val DRIVER_CREDENTIALS_CA_CERT_PATH =
    s"$DRIVER_CREDENTIALS_SECRETS_BASE_DIR/$DRIVER_CREDENTIALS_CA_CERT_SECRET_NAME"
  val DRIVER_CREDENTIALS_CLIENT_KEY_SECRET_NAME = "client-key"
  val DRIVER_CREDENTIALS_CLIENT_KEY_PATH =
    s"$DRIVER_CREDENTIALS_SECRETS_BASE_DIR/$DRIVER_CREDENTIALS_CLIENT_KEY_SECRET_NAME"
  val DRIVER_CREDENTIALS_CLIENT_CERT_SECRET_NAME = "client-cert"
  val DRIVER_CREDENTIALS_CLIENT_CERT_PATH =
    s"$DRIVER_CREDENTIALS_SECRETS_BASE_DIR/$DRIVER_CREDENTIALS_CLIENT_CERT_SECRET_NAME"
  val DRIVER_CREDENTIALS_OAUTH_TOKEN_SECRET_NAME = "oauth-token"
  val DRIVER_CREDENTIALS_OAUTH_TOKEN_PATH =
    s"$DRIVER_CREDENTIALS_SECRETS_BASE_DIR/$DRIVER_CREDENTIALS_OAUTH_TOKEN_SECRET_NAME"
  val DRIVER_CREDENTIALS_SECRET_VOLUME_NAME = "kubernetes-credentials"

  // Default and fixed ports
  val DEFAULT_DRIVER_PORT = 7078
  val DEFAULT_BLOCKMANAGER_PORT = 7079
  val DRIVER_PORT_NAME = "driver-rpc-port"
  val BLOCK_MANAGER_PORT_NAME = "blockmanager"
  val UI_PORT_NAME = "spark-ui"

  // Environment Variables
  val ENV_DRIVER_URL = "SPARK_DRIVER_URL"
  val ENV_EXECUTOR_CORES = "SPARK_EXECUTOR_CORES"
  val ENV_EXECUTOR_MEMORY = "SPARK_EXECUTOR_MEMORY"
  val ENV_EXECUTOR_DIRS = "SPARK_EXECUTOR_DIRS"
  val ENV_APPLICATION_ID = "SPARK_APPLICATION_ID"
  val ENV_EXECUTOR_ID = "SPARK_EXECUTOR_ID"
  val ENV_EXECUTOR_POD_IP = "SPARK_EXECUTOR_POD_IP"
  val ENV_EXECUTOR_POD_NAME = "SPARK_EXECUTOR_POD_NAME"
  val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
  val ENV_CLASSPATH = "SPARK_CLASSPATH"
  val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
  val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
  val ENV_SPARK_USER = "SPARK_USER"
  val ENV_RESOURCE_PROFILE_ID = "SPARK_RESOURCE_PROFILE_ID"
  // Spark app configs for containers
  val SPARK_CONF_VOLUME_DRIVER = "spark-conf-volume-driver"
  val SPARK_CONF_VOLUME_EXEC = "spark-conf-volume-exec"
  val SPARK_CONF_DIR_INTERNAL = "/opt/spark/conf"
  val SPARK_CONF_FILE_NAME = "spark.properties"
  val SPARK_CONF_PATH = s"$SPARK_CONF_DIR_INTERNAL/$SPARK_CONF_FILE_NAME"
  val ENV_HADOOP_TOKEN_FILE_LOCATION = "HADOOP_TOKEN_FILE_LOCATION"

  // BINDINGS
  val ENV_PYSPARK_PYTHON = "PYSPARK_PYTHON"
  val ENV_PYSPARK_DRIVER_PYTHON = "PYSPARK_DRIVER_PYTHON"

  // Pod spec templates
  val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml"
  val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH = "/opt/spark/pod-template"
  val POD_TEMPLATE_VOLUME = "pod-template-volume"
  val POD_TEMPLATE_CONFIGMAP = "driver-podspec-conf-map"
  val POD_TEMPLATE_KEY = "podspec-configmap-key"

  // Miscellaneous
  val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc"
  val DEFAULT_DRIVER_CONTAINER_NAME = "spark-kubernetes-driver"
  val DEFAULT_EXECUTOR_CONTAINER_NAME = "spark-kubernetes-executor"
  val NON_JVM_MEMORY_OVERHEAD_FACTOR = 0.4d

  // Hadoop Configuration
  val HADOOP_CONF_VOLUME = "hadoop-properties"
  val KRB_FILE_VOLUME = "krb5-file"
  val HADOOP_CONF_DIR_PATH = "/opt/hadoop/conf"
  val KRB_FILE_DIR_PATH = "/etc"
  val ENV_HADOOP_CONF_DIR = "HADOOP_CONF_DIR"
  val HADOOP_CONFIG_MAP_NAME =
    "spark.kubernetes.executor.hadoopConfigMapName"

  // Kerberos Configuration
  val KERBEROS_DT_SECRET_NAME =
    "spark.kubernetes.kerberos.dt-secret-name"
  val KERBEROS_DT_SECRET_KEY =
    "spark.kubernetes.kerberos.dt-secret-key"
  val KERBEROS_SECRET_KEY = "hadoop-tokens"
  val KERBEROS_KEYTAB_VOLUME = "kerberos-keytab"
  val KERBEROS_KEYTAB_MOUNT_POINT = "/mnt/secrets/kerberos-keytab"

  // Hadoop credentials secrets for the Spark app.
  val SPARK_APP_HADOOP_CREDENTIALS_BASE_DIR = "/mnt/secrets/hadoop-credentials"
  val SPARK_APP_HADOOP_SECRET_VOLUME_NAME = "hadoop-secret"

  // Application resource types.
  val APP_RESOURCE_TYPE_JAVA = "java"
  val APP_RESOURCE_TYPE_PYTHON = "python"
  val APP_RESOURCE_TYPE_R = "r"
}

相关信息

spark 源码目录

相关文章

spark Config 源码

spark KubernetesConf 源码

spark KubernetesDriverSpec 源码

spark KubernetesExecutorSpec 源码

spark KubernetesUtils 源码

spark KubernetesVolumeSpec 源码

spark KubernetesVolumeUtils 源码

spark SparkKubernetesClientFactory 源码

spark SparkPod 源码