Deploy Alluxio on Kubernetes

Alluxio can be run on Kubernetes. This guide demonstrates how to run Alluxio on Kubernetes using the specification included in the Alluxio Docker image or helm.

• Prerequisites
• Basic Setup
  • Deploy
  • Verify
  • Access the Web UI
• Advanced Setup
  • Enable Short-circuit Access
    • Option1: Use local mode
    • Option2: Use uuid (default)
  • Verify Short-circuit Operations
  • Disable Short-Circuit Operations
  • Enable remote logging
  • POSIX API
    • CSI
  • Toggle Master or Worker in Helm chart
  • Kubernetes Configuration Options
    • ServiceAccounts
    • Node Selectors & Tolerations
    • Host Aliases
    • Deployment Strategy
    • ImagePullSecrets
• Troubleshooting

Prerequisites

• A Kubernetes cluster (version 1.11+) with Beta feature gate APIs enabled
  • The Alluxio Helm chart which the Kubernetes resource specifications are built from supports Kubernetes version 1.11+.
  • Beta feature gates are enabled by default for Kubernetes cluster installations
• Cluster access to an Alluxio Docker image alluxio/alluxio. If using a private Docker registry, refer to the Kubernetes private image registry documentation.
• Ensure the cluster’s Kubernetes Network Policy allows for connectivity between applications (Alluxio clients) and the Alluxio Pods on the defined ports.

Basic Setup

This tutorial walks through a basic Alluxio setup on Kubernetes. Alluxio supports two methods of installation on Kubernetes: either using helm charts or using kubectl. When available, helm is the preferred way to install Alluxio. If helm is not available or if additional deployment customization is desired, kubectl can be used directly using native Kubernetes resource specifications.

Note: From Alluxio 2.3 on, Alluxio only supports helm 3. See how to migrate from helm 2 to 3 here.

$ id=$(docker create alluxio/alluxio:2.7.1)
$ docker cp $id:/opt/alluxio/integration/kubernetes/ - > kubernetes.tar
$ docker rm -v $id 1>/dev/null
$ tar -xvf kubernetes.tar
$ cd kubernetes/helm-chart/alluxio

# Name the file alluxio-master-journal-pv.yaml
kind: PersistentVolume
apiVersion: v1
metadata:
  name: alluxio-journal-0
  labels:
    type: local
spec:
  storageClassName: standard
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  hostPath:
    path: /tmp/alluxio-journal-0

$ kubectl create -f alluxio-master-journal-pv.yaml

Deploy

• helm
• kubectl

$ helm repo add alluxio-charts https://alluxio-charts.storage.googleapis.com/openSource/2.7.1

properties:
  alluxio.master.mount.table.root.ufs: "<under_storage_address>"

$ helm inspect values alluxio-charts/alluxio

properties:
  alluxio.master.mount.table.root.ufs: "s3a://<bucket>"
  alluxio.master.mount.table.root.option.aws.accessKeyId: "<accessKey>"
  alluxio.master.mount.table.root.option.aws.secretKey: "<secretKey>"

master:
  count: 1 # For multiMaster mode increase this to >1

journal:
  type: "UFS"
  ufsType: "local"
  folder: "/journal"
  size: 1Gi
  # volumeType controls the type of journal volume.
  # It can be "persistentVolumeClaim" or "emptyDir"
  volumeType: persistentVolumeClaim
  # Attributes to use when the journal is persistentVolumeClaim
  storageClass: "standard"
  accessModes:
    - ReadWriteOnce

master:
  count: 1 # For multiMaster mode increase this to >1

journal:
  type: "UFS"
  ufsType: "local"
  folder: "/journal"
  size: 1Gi
  # volumeType controls the type of journal volume.
  # It can be "persistentVolumeClaim" or "emptyDir"
  volumeType: emptyDir
  # Attributes to use when the journal is emptyDir
  medium: ""

$ kubectl create secret generic alluxio-hdfs-config --from-file=${HADOOP_CONF_DIR}/core-site.xml --from-file=${HADOOP_CONF_DIR}/hdfs-site.xml

journal:
  type: "UFS"
  ufsType: "HDFS"
  folder: "hdfs://{$hostname}:{$hostport}/journal"

properties:
  alluxio.master.mount.table.root.ufs: "hdfs://<ns>"
  alluxio.master.journal.ufs.option.alluxio.underfs.hdfs.configuration: "/secrets/hdfsConfig/core-site.xml:/secrets/hdfsConfig/hdfs-site.xml"

secrets:
  master:
    alluxio-hdfs-config: hdfsConfig
  worker:
    alluxio-hdfs-config: hdfsConfig

master:
  count: 3

journal:
  type: "EMBEDDED"
  folder: "/journal"
  # volumeType controls the type of journal volume.
  # It can be "persistentVolumeClaim" or "emptyDir"
  volumeType: persistentVolumeClaim
  size: 1Gi
  # Attributes to use when the journal is persistentVolumeClaim
  storageClass: "standard"
  accessModes:
    - ReadWriteOnce

master:
  count: 3

journal:
  type: "UFS"
  ufsType: "local"
  folder: "/journal"
  size: 1Gi
  # volumeType controls the type of journal volume.
  # It can be "persistentVolumeClaim" or "emptyDir"
  volumeType: emptyDir
  # Attributes to use when the journal is emptyDir
  medium: ""

$ kubectl create secret generic alluxio-hdfs-config --from-file=${HADOOP_CONF_DIR}/core-site.xml --from-file=${HADOOP_CONF_DIR}/hdfs-site.xml

properties:
  alluxio.master.mount.table.root.ufs: "hdfs://<ns>"
  alluxio.master.mount.table.root.option.alluxio.underfs.hdfs.configuration: "/secrets/hdfsConfig/core-site.xml:/secrets/hdfsConfig/hdfs-site.xml"
secrets:
  master:
    alluxio-hdfs-config: hdfsConfig
  worker:
    alluxio-hdfs-config: hdfsConfig

properties:
  alluxio.master.metastore: ROCKS
  alluxio.master.metastore.dir: /metastore

metastore:
  volumeType: persistentVolumeClaim # Options: "persistentVolumeClaim" or "emptyDir"
  size: 1Gi
  mountPath: /metastore
  # Attributes to use when the metastore is persistentVolumeClaim
  storageClass: "standard"
  accessModes:
   - ReadWriteOnce

properties:
  alluxio.master.metastore: ROCKS
  alluxio.master.metastore.dir: /metastore

metastore:
  volumeType: emptyDir # Options: "persistentVolumeClaim" or "emptyDir"
  size: 1Gi
  mountPath: /metastore
  # Attributes to use when the metastore is emptyDir
  medium: ""

secrets:
  master:
    alluxio-hdfs-config: hdfsConfig
    alluxio-ceph-config: cephConfig
  worker:
    alluxio-hdfs-config: hdfsConfig
    alluxio-ceph-config: cephConfig

tieredstore:
  levels:
  - level: 0
    mediumtype: MEM
    path: /dev/shm
    type: emptyDir
    high: 0.95
    low: 0.7

tieredstore:
  levels:
  - level: 0
    mediumtype: MEM
    path: /dev/shm
    type: hostPath
    high: 0.95
    low: 0.7
  - level: 1
    mediumtype: SSD
    path: /ssd-disk
    type: hostPath
    high: 0.95
    low: 0.7

tieredstore:
  levels:
  - level: 0
    mediumtype: MEM
    path: /dev/shm
    type: persistentVolumeClaim
    name: alluxio-mem
    quota: 1G
    high: 0.95
    low: 0.7
  - level: 1
    mediumtype: SSD
    path: /ssd-disk
    type: persistentVolumeClaim
    name: alluxio-ssd
    quota: 10G
    high: 0.95
    low: 0.7

tieredstore:
  levels:
  - level: 0
    mediumtype: MEM,SSD
    path: /dev/shm,/alluxio-ssd
    type: persistentVolumeClaim
    name: alluxio-mem,alluxio-ssd
    quota: 1GB,10GB
    high: 0.95
    low: 0.7

$ helm install alluxio -f config.yaml alluxio-charts/alluxio

# Use the same config.yaml and switch on journal formatting
$ helm upgrade alluxio -f config.yaml --set journal.format.runFormat=true alluxio-charts/alluxio

$ helm install alluxio -f config.yaml --set journal.format.runFormat=true alluxio-charts/alluxio

$ helm delete alluxio

$ id=$(docker create alluxio/alluxio:2.7.1)
$ docker cp $id:/opt/alluxio/integration/kubernetes/ - > kubernetes.tar
$ docker rm -v $id 1>/dev/null
$ tar -xvf kubernetes.tar
$ cd kubernetes

$ cp alluxio-configmap.yaml.template alluxio-configmap.yaml

-Dalluxio.master.mount.table.root.ufs=<under_storage_address>

$ kubectl create -f alluxio-configmap.yaml

$ mv master/alluxio-master-service.yaml.template master/alluxio-master-service.yaml
$ mv master/alluxio-master-statefulset.yaml.template master/alluxio-master-statefulset.yaml

$ mv worker/alluxio-worker-daemonset.yaml.template worker/alluxio-worker-daemonset.yaml

spec:
  hostAliases:
  - ip: "<namenode_ip>"
    hostnames:
    - "<namenode>"

kind: StatefulSet
metadata:
  name: alluxio-master
spec:
  ...
  serviceName: "alluxio-master"
  replicas: 1
  template:
    metadata:
      labels:
        app: alluxio-master
    spec:
      hostAliases:
      - ip: "ip for hdfs-host"
        hostnames:
        - "hdfs-host"

kubectl create secret generic alluxio-hdfs-config --from-file=${HADOOP_CONF_DIR}/core-site.xml --from-file=${HADOOP_CONF_DIR}/hdfs-site.xml

$ kubectl create -f ./master/
$ kubectl create -f ./worker/

- name: journal-format
  image: alluxio/alluxio:2.7.1
  imagePullPolicy: IfNotPresent
  securityContext:
    runAsUser: 1000
  command: ["alluxio","formatJournal"]
  volumeMounts:
    - name: alluxio-journal
      mountPath: /journal

containers:
- name: alluxio-master
  image: alluxio/alluxio:latest
  imagePullPolicy: IfNotPresent
  ...
- name: alluxio-job-master
  image: alluxio/alluxio:latest
  imagePullPolicy: IfNotPresent
  ...

$ kubectl delete daemonset -l app=alluxio

$ kubectl delete service -l app=alluxio
$ kubectl delete statefulset -l app=alluxio

$ kubectl create -f ./master/
$ kubectl create -f ./worker/

# You should see all Alluxio master and worker Pods
$ kubectl get pods

$ kubectl delete -f ./worker/
$ kubectl delete -f ./master/
$ kubectl delete configmap alluxio-config

Verify

If using persistent volumes, the status of the volume(s) should change to CLAIMED and the status of the volume claims should be BOUNDED. You can validate the status of your PersistentVolume and PersistentVolumeClaims using the follow kubectl commands:

$ kubectl get pv
$ kubectl get pvc

• If you have unbound PersistentVolumeClaims, please ensure you have provisioned matching PersistentVolumes. See “(Optional) Provision a Persistent Volume” in Basic Setup.

Once ready, access the Alluxio CLI from the master Pod and run basic I/O tests.

$ kubectl exec -ti alluxio-master-0 /bin/bash

From the master Pod, execute the following:

$ alluxio runTests

Access the Web UI

The Alluxio UI can be accessed from outside the kubernetes cluster using port forwarding.

$ kubectl port-forward alluxio-master-$i <local-port>:19999

The command above allocates a port on the local node <local-port> and forwards traffic on <local-port> to port 19999 of pod alluxio-master-$i. The pod alluxio-master-$i does NOT have to be on the node you are running this command.

Note: i=0 for the the first master Pod. When running multiple masters, forward port for each master. Only the primary master serves the Web UI.

For example, you are on a node with hostname master-node-1 and you would like to serve the Alluxio master web UI for alluxio-master-0 on master-node-1:8080. Here’s the command you can run:

[alice@master-node-1 ~]$ kubectl port-forward --address 0.0.0.0 pods/alluxio-master-0 8080:19999

This forwards the local port master-node-1:8080 to the port on the Pod alluxio-master-0:19999. The Pod alluxio-master-0 does NOT need to be running on master-node-1.

You will see messages like below when there are incoming connections.

[alice@master-node-1 ~]$ kubectl port-forward --address 0.0.0.0 alluxio-master-0 8080:19999
Forwarding from 0.0.0.0:8080 -> 19999
Handling connection for 8080
Handling connection for 8080
Handling connection for 8080
Handling connection for 8080

You can terminate the process to stop the port forwarding, with either Ctrl + C or kill.

For more information about K8s port-forward see the K8s doc.

Advanced Setup

Enable Short-circuit Access

Short-circuit access enables clients to perform read and write operations directly against the worker bypassing the networking interface. For performance-critical applications it is recommended to enable short-circuit operations against Alluxio because it can increase a client’s read and write throughput when co-located with an Alluxio worker.

This feature is enabled by default (see next section to disable this feature), however requires extra configuration to work properly in Kubernetes environments.

There are two modes for using short-circuit.

Option1: Use local mode

In this mode, the Alluxio client and local Alluxio worker recognize each other if the client hostname matches the worker hostname. This is called Hostname Introspection. In this mode, the Alluxio client and local Alluxio worker share the tiered storage of Alluxio worker.

• helm
• kubectl

shortCircuit:
  enabled: true
  policy: local

-Dalluxio.user.short.circuit.enabled=true \
-Dalluxio.worker.data.server.domain.socket.as.uuid=false

Option2: Use uuid (default)

This is the default policy used for short-circuit in Kubernetes.

If the client or worker container is using virtual networking, their hostnames may not match. In such a scenario, set the following property to use filesystem inspection to enable short-circuit operations and make sure the client container mounts the directory specified as the domain socket path. Short-circuit writes are then enabled if the worker UUID is located on the client filesystem.

Domain Socket Path. The domain socket is a volume which should be mounted on:

• All Alluxio workers
• All application containers which intend to read/write through Alluxio

This domain socket volume can be either a PersistentVolumeClaim or a hostPath Volume.

Use PersistentVolumeClaim. By default, this domain socket volume is a PersistentVolumeClaim. You need to provision a PersistentVolume to this PersistentVolumeClaim. And this PersistentVolume should be either local or hostPath.

• helm
• kubectl

# These are the default configurations
shortCircuit:
  enabled: true
  policy: uuid
  size: 1Mi
  # volumeType controls the type of shortCircuit volume.
  # It can be "persistentVolumeClaim" or "hostPath"
  volumeType: persistentVolumeClaim
  # Attributes to use if the domain socket volume is PVC
  pvcName: alluxio-worker-domain-socket
  accessModes:
    - ReadWriteOnce
  storageClass: standard

-Dalluxio.worker.data.server.domain.socket.address=/opt/domain -Dalluxio.worker.data.server.domain.socket.as.uuid=true

volumes:
  - name: alluxio-domain
    persistentVolumeClaim:
      claimName: "alluxio-worker-domain-socket"

Use hostPath Volume. You can also directly define the workers to use a hostPath Volume for domain socket.

• helm
• kubectl

shortCircuit:
  enabled: true
  policy: uuid
  size: 1Mi
  # volumeType controls the type of shortCircuit volume.
  # It can be "persistentVolumeClaim" or "hostPath"
  volumeType: hostPath
  # Attributes to use if the domain socket volume is hostPath
  hostPath: "/tmp/alluxio-domain" # The hostPath directory to use

volumes:
  - name: alluxio-domain
    hostPath:
      path: /tmp/alluxio-domain
      type: DirectoryOrCreate

Verify Short-circuit Operations

To verify short-circuit reads and writes monitor the metrics displayed under:

1. the metrics tab of the web UI as Domain Socket Alluxio Read and Domain Socket Alluxio Write
2. or, the metrics json as cluster.BytesReadDomain and cluster.BytesWrittenDomain
3. or, the fsadmin metrics CLI as Short-circuit Read (Domain Socket) and Alluxio Write (Domain Socket)

Disable Short-Circuit Operations

To disable short-circuit operations, the operation depends on how you deploy Alluxio.

Note: As mentioned, disabling short-circuit access for Alluxio workers will result in worse I/O throughput

• helm
• kubectl

shortCircuit:
  enabled: false

-Dalluxio.user.short.circuit.enabled=false

Enable remote logging

Alluxio supports a centralized log server that collects logs for all Alluxio processes. You can find the specific section at Remote logging. This can be enabled on K8s too, so that all Alluxio pods will send logs to this log server.

• helm
• kubectl

logserver:
  enabled: true

logserver:
  enabled: true
  # volumeType controls the type of log volume.
  # It can be "persistentVolumeClaim" or "hostPath" or "emptyDir"
  volumeType: emptyDir
  # Attributes to use when the log volume is emptyDir
  medium: ""
  size: 4Gi

logserver:
  enabled: true
  # volumeType controls the type of log volume.
  # It can be "persistentVolumeClaim" or "hostPath" or "emptyDir"
  volumeType: persistentVolumeClaim
  # Attributes to use if the log volume is PVC
  pvcName: alluxio-logserver-logs
  accessModes:
    - ReadWriteOnce
  storageClass: standard
  # If you are dynamically provisioning PVs, the selector on the PVC should be empty.
  # Ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1
  selector: {}
  # If you are manually allocating PV for the logserver,
  # it is recommended to use selectors to make sure the PV and PVC match as expected.
  # You can specify selectors like below:
  # Example:
  # selector:
  #   matchLabels:
  #     role: alluxio-logserver
  #     app: alluxio
  #     chart: alluxio-<chart version>
  #     release: alluxio
  #     heritage: Helm
  #     dc: data-center-1
  #     region: us-east

apiVersion: v1
kind: ConfigMap
metadata:
  ..omitted
data:
  ..omitted
  ALLUXIO_LOGSERVER_HOSTNAME: alluxio-logserver
  ALLUXIO_LOGSERVER_PORT: "45600"

$ cp logserver/alluxio-logserver-deployment.yaml.template logserver/alluxio-logserver-deployment.yaml

  volumes:      
  - name: alluxio-logs
    emptyDir:
      medium: 
      sizeLimit: "4Gi"

  volumeMounts:
  - name: alluxio-logs
    mountPath: /opt/alluxio/logs

  volumes:      
  - name: alluxio-logs
    persistentVolumeClaim:
      claimName: "alluxio-logserver-logs"

$ cp logserver/alluxio-logserver-pvc.yaml.template logserver/alluxio-logserver-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: alluxio-logserver-logs
  ..omitted
spec:
  volumeMode: Filesystem
  resources:
    requests:
      storage: 4Gi
  storageClassName: standard
  accessModes:
    - ReadWriteOnce
  # If you are using dynamic provisioning, leave the selector empty.
  selector: {}
  # If you are manually allocating PV for the logserver,
  # it is recommended to use selectors to make sure the PV and PVC match as expected.
  # You can specify selectors like below:
  # Example:
  # selector:
  #   matchLabels:
  #     role: alluxio-logserver
  #     app: alluxio
  #     chart: alluxio-<chart version>
  #     release: alluxio
  #     heritage: Helm
  #     dc: data-center-1
  #     region: us-east

$ kubectl create -f alluxio-logserver-pvc.yaml

$ kubectl create -f alluxio-logserver-deployment.yaml

$ cp logserver/alluxio-logserver-service.yaml.template logserver/alluxio-logserver-service.yaml
$ kubectl create -f logserver/alluxio-logserver-service.yaml

Verify log server

You can go into the log server pod and verify the logs exist.

$ kubectl exec -it <logserver-pod-name> bash
# In the logserver pod
bash-4.4$ pwd
/opt/alluxio
# You should see logs collected from other Alluxio pods
bash-4.4$ ls -al logs
total 16
drwxrwsr-x    4 1001     bin           4096 Jan 12 03:14 .
drwxr-xr-x    1 alluxio  alluxio         18 Jan 12 02:38 ..
drwxr-sr-x    2 alluxio  bin           4096 Jan 12 03:14 job_master
-rw-r--r--    1 alluxio  bin            600 Jan 12 03:14 logserver.log
drwxr-sr-x    2 alluxio  bin           4096 Jan 12 03:14 master
drwxr-sr-x    2 alluxio  bin           4096 Jan 12 03:14 worker
drwxr-sr-x    2 alluxio  bin           4096 Jan 12 03:14 job_worker

POSIX API

Once Alluxio is deployed on Kubernetes, there are multiple ways in which a client application can connect to it. For applications using the POSIX API, application containers can simply mount the Alluxio FileSystem.

In order to use the POSIX API, first deploy the Alluxio FUSE daemon.

• helm
• kubectl

fuse:
  enabled: true
  clientEnabled: true

fuse:
  enabled: true
  clientEnabled: true
  mountPath: /mnt/alluxio-fuse

$ helm upgrade alluxio -f config.yaml \
  --set fuse.enabled=true \
  --set fuse.clientEnabled=true \
  alluxio-charts/alluxio

$ cp alluxio-fuse.yaml.template alluxio-fuse.yaml
$ kubectl create -f alluxio-fuse.yaml

$ cp alluxio-fuse-client.yaml.template alluxio-fuse-client.yaml
$ kubectl create -f alluxio-fuse-client.yaml

CSI

Other than using Alluxio FUSE daemon, you could also use CSI to mount the Alluxio FileSystem into application containers.

In order to use CSI, you need a Kubernetes cluster with version at least 1.17, with RBAC enabled in API Server.

Step 1: Customize configurations and generate templates

You can either use the default CSI configurations provided in here under the csi section, or you can customize them to make it suitable for your workload. Here are some common properties that you can customize:

Then please use helm-generate.sh (see here for usage) to generate related templates. All CSI related templates will be under ${ALLUXIO_HOME}/integration/kubernetes/<deploy-mode>/csi.

Step 2: Deploy CSI services

Modify or add any configuration properties as required, then create the respective resources.

$ mv alluxio-csi-controller-rbac.yaml.template alluxio-csi-controller-rbac.yaml
$ mv alluxio-csi-controller.yaml.template alluxio-csi-controller.yaml
$ mv alluxio-csi-driver.yaml.template alluxio-csi-driver.yaml
$ mv alluxio-csi-nodeplugin.yaml.template alluxio-csi-nodeplugin.yaml

Then run

$ kubectl apply -f alluxio-csi-controller-rbac.yaml -f alluxio-csi-controller.yaml -f alluxio-csi-driver.yaml -f alluxio-csi-nodeplugin.yaml

to deploy CSI-related services.

Step 3: Provisioning

We provide both templates for k8s dynamic provisioning and static provisioning. Please choose the suitable provisioning methods according to your use case. You can refer to Persistent Volumes | Kubernetes and Dynamic Volume Provisioning | Kubernetes to get more details.

• Persistent Volumes
• Dynamic Volume Provisioning

$ mv alluxio-pv.yaml.template alluxio-pv.yaml
$ mv alluxio-pvc-static.yaml.template alluxio-pvc-static.yaml

$ kubectl apply -f alluxio-pv.yaml
$ kubectl apply -f alluxio-pvc-static.yaml

$ mv alluxio-storage-class.yaml.template alluxio-storage-class.yaml
$ mv alluxio-pvc.yaml.template alluxio-pvc.yaml

$ kubectl apply -f alluxio-storage-class.yaml
$ kubectl apply -f alluxio-pvc.yaml

Step 4: Deploy applications

Now you can put the PVC name in your application pod spec to use the Alluxio FileSystem. The template alluxio-nginx-pod.yaml.template shows how to use PVC in the pod. You can also deploy it by running

$ mv alluxio-nginx-pod.yaml.templte alluxio-nginx-pod.yaml
$ kubectl apply -f alluxio-nginx-pod.yaml

to validate that CSI has been deployed, and you can successfully access the data stored in Alluxio.

For more information on how to configure a pod to use a persistent volume for storage in Kubernetes, please refer to here.

Toggle Master or Worker in Helm chart

In use cases where you wish to install Alluxio masters and workers separately with the Helm chart, use the following respective toggles:

master:
  enabled: false

worker:
  enabled: false

Kubernetes Configuration Options

The following options are provided in our Helm chart as additional parameters for experienced Kubernetes users.

ServiceAccounts

By default Kubernetes will assign the namespace’s default ServiceAccount to new pods in a namespace. You may specify for Alluxio pods to use any existing ServiceAccounts you may have in your cluster through the following:

• helm
• kubectl

serviceAccount: sa-alluxio

master:
  serviceAccount: sa-alluxio-master

worker:
  serviceAccount: sa-alluxio-worker

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: alluxio-master
spec:
  template:
    spec:
      serviceAccountName: sa-alluxio

Node Selectors & Tolerations

Kubernetes provides many options to control the scheduling of pods onto nodes in the cluster. The most direct of which is a node selector.

However, Kubernetes will avoid scheduling pods on any tainted nodes. To allow certain pods to schedule on such nodes, Kubernetes allows you to specify tolerations for those taints. See the Kubernetes documentation on taints and tolerations for more details.

• helm
• kubectl

nodeSelector: {"app": "alluxio"}

tolerations: [ {"key": "env", "operator": "Equal", "value": "prod", "effect": "NoSchedule"} ]

master:
  nodeSelector: {"app": "alluxio-master"}

worker:
  nodeSelector: {"app": "alluxio-worker"}

logserver:
  tolerations: [ {"key": "app", "operator": "Equal", "value": "logging", "effect": "NoSchedule"} ]

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: alluxio-master
spec:
  template:
    spec:
      nodeSelector:
        app: alluxio
      tolerations:
        - effect: NoSchedule
          key: env
          operator: Equal
          value: prod

Host Aliases

If you wish to add or override hostname resolution in the pods, Kubernetes exposes the containers’ /etc/hosts file via host aliases. This can be particularly useful for providing hostname addresses for services not managed by Kubernetes, like HDFS.

• helm
• kubectl

hostAliases:
- ip: "127.0.0.1"
  hostnames:
    - "foo.local"
    - "bar.local"
- ip: "10.1.2.3"
  hostnames:
    - "foo.remote"
    - "bar.remote"

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: alluxio-master
spec:
  template:
    spec:
      hostAliases:
      - ip: "127.0.0.1"
        hostnames:
          - "foo.local"
          - "bar.local"
      - ip: "10.1.2.3"
        hostnames:
          - "foo.remote"
          - "bar.local"

Deployment Strategy

By default Kubernetes will use the ‘RollingUpdate’ deployment strategy to progressively upgrade Pods when changes are detected.

• helm
• kubectl

logserver:
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%
      maxSurge: 1

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: alluxio-master
spec:
  template:
    spec:
      strategy:
        type: Recreate

ImagePullSecrets

Kubernetes supports accessing images from a Private Registry. After creating the registry credentials Secret in Kubernetes, you pass the secret to your Pods via imagePullSecrets.

• helm
• kubectl

imagePullSecrets:
  - ecr
  - dev

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: alluxio-master
spec:
  template:
    spec:
      containers:
      - name: alluxio-master
        image: private-registry/alluxio:2.7.1
      imagePullSecrets:
      - name: ecr
      - name: dev

Troubleshooting

Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Host is unreachable: <host>/<IP>:29999

$ kubectl exec -ti alluxio-master-0 /bin/bash

$ alluxio logLevel --level DEBUG --logName alluxio

$ kubectl logs -f alluxio-master-0 -c alluxio-master

$ kubectl logs -f alluxio-worker-<id> -c alluxio-worker

$ kubectl logs -f alluxio-master-0 -c alluxio-job-master

$ kubectl logs -f alluxio-worker-<id> -c alluxio-job-worker

2020-07-27 21:39:06,030 ERROR GrpcDataServer - Alluxio worker gRPC server failed to start on /opt/domain/1d6d7c85-dee0-4ac5-bbd1-86eb496a2a50
java.io.IOException: Failed to bind
	at io.grpc.netty.NettyServer.start(NettyServer.java:252)
	at io.grpc.internal.ServerImpl.start(ServerImpl.java:184)
	at io.grpc.internal.ServerImpl.start(ServerImpl.java:90)
	at alluxio.grpc.GrpcServer.lambda$start$0(GrpcServer.java:77)
	at alluxio.retry.RetryUtils.retry(RetryUtils.java:39)
	at alluxio.grpc.GrpcServer.start(GrpcServer.java:77)
	at alluxio.worker.grpc.GrpcDataServer.<init>(GrpcDataServer.java:107)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at alluxio.util.CommonUtils.createNewClassInstance(CommonUtils.java:273)
	at alluxio.worker.DataServer$Factory.create(DataServer.java:47)
	at alluxio.worker.AlluxioWorkerProcess.<init>(AlluxioWorkerProcess.java:162)
	at alluxio.worker.WorkerProcess$Factory.create(WorkerProcess.java:46)
	at alluxio.worker.WorkerProcess$Factory.create(WorkerProcess.java:38)
	at alluxio.worker.AlluxioWorker.main(AlluxioWorker.java:72)
Caused by: io.netty.channel.unix.Errors$NativeIoException: bind(..) failed: Filename too long

$ kubectl get po -o json alluxio-worker-xxxxx | jq '.spec.containers[].resources'
{
  "limits": {
    "cpu": "4",
    "memory": "4G"
  },
  "requests": {
    "cpu": "1",
    "memory": "2G"
  }
}
{
  "limits": {
    "cpu": "4",
    "memory": "4G"
  },
  "requests": {
    "cpu": "1",
    "memory": "1G"
  }
}

worker:
  resources:
    limits:
      cpu: "4"
      memory: "4G"
    requests:
      cpu: "1"
      memory: "2G"

jobWorker:
  resources:
    limits:
      cpu: "4"
      memory: "4G"
    requests:
      cpu: "1"
      memory: "1G"

$ kubectl get no -o json k8sworkernode-0 | jq '.status.allocatable'
{
  "cpu": "8",
  "ephemeral-storage": "123684658586",
  "hugepages-1Gi": "0",
  "hugepages-2Mi": "0",
  "memory": "64886128Ki",
  "pods": "110"
}

properties:
  alluxio.worker.ramdisk.size: 2G
  alluxio.worker.tieredstore.levels: 1
  alluxio.worker.tieredstore.level0.alias: MEM
  alluxio.worker.tieredstore.level0.dirs.mediumtype: MEM
  alluxio.worker.tieredstore.level0.dirs.path: /dev/shm
  alluxio.worker.tieredstore.level0.dirs.quota: 2G

$ kubectl get cm -o json alluxio-config | jq '.data.ALLUXIO_WORKER_JAVA_OPTS'
"-Dalluxio.worker.ramdisk.size=2G
-Dalluxio.worker.tieredstore.levels=1
-Dalluxio.worker.tieredstore.level0.alias=MEM
-Dalluxio.worker.tieredstore.level0.dirs.mediumtype=MEM
-Dalluxio.worker.tieredstore.level0.dirs.path=/dev/shm
-Dalluxio.worker.tieredstore.level0.dirs.quota=2G "

spec:
  template:
    spec:
      volumes:
        - name: mem
          emptyDir:
            medium: "Memory"
            sizeLimit: 1G

apiVersion: v1
kind: ConfigMap
metadata:
  name: alluxio-config
data:
  ALLUXIO_JAVA_OPTS: |-
    -Xmx2g -Dalluxio.master.hostname=alluxio-master-0 ...
  ALLUXIO_MASTER_JAVA_OPTS: |-
    -Dalluxio.master.hostname=${ALLUXIO_MASTER_HOSTNAME}
  ALLUXIO_JOB_MASTER_JAVA_OPTS: |-
    -Dalluxio.master.hostname=${ALLUXIO_MASTER_HOSTNAME}
  ALLUXIO_WORKER_JAVA_OPTS: |-
    -XX:MaxDirectMemorySize=2g -Dalluxio.worker.hostname=${ALLUXIO_WORKER_HOSTNAME} ...
  ALLUXIO_JOB_WORKER_JAVA_OPTS: |-
    -XX:MaxDirectMemorySize=1g -Dalluxio.worker.hostname=${ALLUXIO_WORKER_HOSTNAME} ...
  ALLUXIO_FUSE_JAVA_OPTS: |-
    -Dalluxio.user.hostname=${ALLUXIO_CLIENT_HOSTNAME} -XX:MaxDirectMemorySize=2g
  ALLUXIO_WORKER_TIEREDSTORE_LEVEL0_DIRS_PATH: /dev/shm

# Recommended JVM Heap options for running in Docker
# Ref: https://developers.redhat.com/blog/2017/03/14/java-inside-docker/
# These JVM options are common to all Alluxio services
jvmOptions:
  - "-XX:+UnlockExperimentalVMOptions"
  - "-XX:+UseCGroupMemoryLimitForHeap"
  - "-XX:MaxRAMFraction=2"