Nvidia 雲原生 GPU Operator 分析¶

sudo mkdir /etc/rancher/rke2 -p

cat <<EOF | sudo tee /etc/rancher/rke2/config.yaml
write-kubeconfig-mode: "0644"
write-kubeconfig: "/root/.kube/config"
cni: "calico"
tls-san:
  - dxlab-nb-00
  - 192.168.50.195
EOF

sudo apt install curl -y

curl -sfL https://get.rke2.io --output install.sh

chmod +x install.sh

sudo ./install.sh

# Enable and activate RKE2 server
sudo systemctl enable rke2-server.service
sudo systemctl start rke2-server.service

sudo systemctl status rke2-server

● rke2-server.service - Rancher Kubernetes Engine v2 (server)
     Loaded: loaded (/usr/local/lib/systemd/system/rke2-server.service; enabled; vendor preset: enabled)
     Active: active (running) since Thu 2023-01-26 22:24:54 EST; 28s ago
       Docs: https://github.com/rancher/rke2#readme
    Process: 7423 ExecStartPre=/bin/sh -xc ! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service (code=exited, status=0/SUCCESS)
    Process: 7425 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS)
    Process: 7428 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
   Main PID: 7429 (rke2)
      Tasks: 153
     Memory: 1.8G
     CGroup: /system.slice/rke2-server.service
             ├─7429 /usr/local/bin/rke2 server
             ├─7451 containerd -c /var/lib/rancher/rke2/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd 
             ...
             ...

# copy RKE2 kubeconfig file to the default location
mkdir ~/.kube
sudo cp /etc/rancher/rke2/rke2.yaml ~/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
chmod 600 ~/.kube/config

# add RKE2 binaries to path
export PATH=$PATH:/var/lib/rancher/rke2/bin
echo "export PATH=$PATH:/var/lib/rancher/rke2/bin" >> ~/.bashrc

export KUBECONFIG=$HOME/.kube/config
echo "export KUBECONFIG=$HOME/.kube/config" | tee -a ~/.bashrc

kubectl get nodes

NAME          STATUS   ROLES                       AGE     VERSION
dxlab-nb-00   Ready    control-plane,etcd,master   6m23s   v1.24.9+rke2r2

sudo cat /var/lib/rancher/rke2/agent/etc/containerd/config.toml

version = 2

[plugins."io.containerd.internal.v1.opt"]
  path = "/var/lib/rancher/rke2/agent/containerd"
[plugins."io.containerd.grpc.v1.cri"]
  stream_server_address = "127.0.0.1"
  stream_server_port = "10010"
  enable_selinux = false
  enable_unprivileged_ports = true
  enable_unprivileged_icmp = true
  sandbox_image = "index.docker.io/rancher/pause:3.6"

[plugins."io.containerd.grpc.v1.cri".containerd]
  snapshotter = "overlayfs"
  disable_snapshot_annotations = true



[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true

sudo apt update

sudo apt search nvidia-driver -y

sudo apt install nvidia-driver-525 nvidia-dkms-525

sudo shutdown now -r

nvidia-smi

Fri Jan 20 01:05:54 2023       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 525.78.01    Driver Version: 525.78.01    CUDA Version: 12.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ...  Off  | 00000000:02:00.0 Off |                  N/A |
| N/A   38C    P8    N/A /  N/A |      4MiB /  2048MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A       942      G   /usr/lib/xorg/Xorg                  4MiB |
+-----------------------------------------------------------------------------+

distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
    && curl -s -L https://nvidia.github.io/libnvidia-container/gpgkey | sudo apt-key add - \
    && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

sudo apt-get update \
    && sudo apt-get install -y nvidia-container-toolkit

sudo systemctl restart rke2-server.service

sudo cat /var/lib/rancher/rke2/agent/etc/containerd/config.toml

version = 2

[plugins."io.containerd.internal.v1.opt"]
  path = "/var/lib/rancher/rke2/agent/containerd"
[plugins."io.containerd.grpc.v1.cri"]
  stream_server_address = "127.0.0.1"
  stream_server_port = "10010"
  enable_selinux = false
  enable_unprivileged_ports = true
  enable_unprivileged_icmp = true
  sandbox_image = "index.docker.io/rancher/pause:3.6"

[plugins."io.containerd.grpc.v1.cri".containerd]
  snapshotter = "overlayfs"
  disable_snapshot_annotations = true



[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true


[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."nvidia"]
  runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."nvidia".options]
  BinaryName = "/usr/bin/nvidia-container-runtime"

docker run -d --name gpu-test -e NVIDIA_VISIBLE_DEVICES=0,1 centos:7 sleep 5000

kubectl describe node/dxlab-nb-00

kubectl apply -f -<<EOF
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: nvidia
handler: nvidia
EOF

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh

helm repo add nvdp https://nvidia.github.io/k8s-device-plugin

helm repo update

helm upgrade -i nvdp nvdp/nvidia-device-plugin \
  --namespace nvidia-device-plugin \
  --create-namespace \
  --version 0.13.0 \
  --set runtimeClassName=nvidia

kubectl get all -n nvidia-device-plugin

NAME                                       DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/nvdp-nvidia-device-plugin   0         0         0       0            0           <none>          22s

helm upgrade -i nvdp nvdp/nvidia-device-plugin \
  --namespace nvidia-device-plugin \
  --create-namespace \
  --version 0.13.0 \
  --set runtimeClassName=nvidia \
  --set gfd.enabled=true

kubectl get all -n nvidia-device-plugin

NAME                                                      READY   STATUS    RESTARTS   AGE
pod/nvdp-gpu-feature-discovery-bvh8w                      1/1     Running   0          69s
pod/nvdp-node-feature-discovery-master-8488dd66f6-l25nl   1/1     Running   0          13m
pod/nvdp-node-feature-discovery-worker-bsct9              1/1     Running   0          13m
pod/nvdp-nvidia-device-plugin-dzmb5                       1/1     Running   0          88s

NAME                                         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)    AGE
service/nvdp-node-feature-discovery-master   ClusterIP   10.43.18.253   <none>        8080/TCP   13m

NAME                                                DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/nvdp-gpu-feature-discovery           1         1         1       1            1           <none>          3m53s
daemonset.apps/nvdp-node-feature-discovery-worker   1         1         1       1            1           <none>          13m
daemonset.apps/nvdp-nvidia-device-plugin            1         1         1       1            1           <none>          35m

NAME                                                 READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/nvdp-node-feature-discovery-master   1/1     1            1           13m

NAME                                                            DESIRED   CURRENT   READY   AGE
replicaset.apps/nvdp-node-feature-discovery-master-8488dd66f6   1         1         1       13m

kubectl describe node/dxlab-nb-00

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

helm repo update 

grafana:
  sidecar:
    dashboards:
      logLevel: "DEBUG"
      # Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana
      enabled: true
      searchNamespace: "ALL"


prometheus:
  prometheusSpec:
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false
    ruleSelectorNilUsesHelmValues: false
    probeSelectorNilUsesHelmValues: false

helm upgrade --install --create-namespace --namespace monitoring  \
     kube-stack-prometheus prometheus-community/kube-prometheus-stack \
     --values kube-stack-prometheus-values.yaml

helm repo add nvdcgm https://nvidia.github.io/dcgm-exporter/helm-charts

helm repo update

helm upgrade -i nvdcgm nvdcgm/dcgm-exporter \
  --namespace nvidia-dcgm \
  --create-namespace 

kubectl get all -n nvidia-dcgm

NAME                             READY   STATUS             RESTARTS         AGE
pod/nvdcgm-dcgm-exporter-xkxt7   0/1     CrashLoopBackOff   11 (2m55s ago)   35m

NAME                           TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
service/nvdcgm-dcgm-exporter   ClusterIP   10.43.190.247   <none>        9400/TCP   35m

NAME                                  DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/nvdcgm-dcgm-exporter   1         1         0       1            0           <none>          35m

kubectl edit daemonset.apps/nvdcgm-dcgm-exporter -n nvidia-dcgm

apiVersion: apps/v1
kind: DaemonSet
metadata:
  ...
  ...
spec:
  selector:
    matchLabels:
      app.kubernetes.io/component: dcgm-exporter
      app.kubernetes.io/instance: nvdcgm
      app.kubernetes.io/name: dcgm-exporter
  template:
    metadata:
      creationTimestamp: null
      labels:
        app.kubernetes.io/component: dcgm-exporter
        app.kubernetes.io/instance: nvdcgm
        app.kubernetes.io/name: dcgm-exporter
    spec:
      containers:
      - args:
        - -f
        - /etc/dcgm-exporter/dcp-metrics-included.csv
        env:
        - name: DCGM_EXPORTER_KUBERNETES
          value: "true"
        - name: DCGM_EXPORTER_LISTEN
          value: :9400
        image: nvcr.io/nvidia/k8s/dcgm-exporter:3.1.3-3.1.2-ubuntu20.04
        imagePullPolicy: IfNotPresent
        ...
        ...
      dnsPolicy: ClusterFirst
      priorityClassName: system-node-critical
      restartPolicy: Always
      runtimeClassName: nvidia
      schedulerName: default-scheduler
      ...
      ...

kubectl get all -n nvidia-dcgm

NAME                             READY   STATUS    RESTARTS   AGE
pod/nvdcgm-dcgm-exporter-qkjlt   1/1     Running   0          17m

NAME                           TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
service/nvdcgm-dcgm-exporter   ClusterIP   10.43.190.247   <none>        9400/TCP   59m

NAME                                  DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/nvdcgm-dcgm-exporter   1         1         1       1            1           <none>          59m

kubectl get service/nvdcgm-dcgm-exporter -n nvidia-dcgm -o yaml

apiVersion: v1
kind: Service
metadata:
  annotations:
    meta.helm.sh/release-name: nvdcgm
    meta.helm.sh/release-namespace: nvidia-dcgm
  creationTimestamp: "2023-01-27T11:27:11Z"
  labels:
    app.kubernetes.io/component: dcgm-exporter
    app.kubernetes.io/instance: nvdcgm
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: dcgm-exporter
    app.kubernetes.io/version: 3.1.2
    helm.sh/chart: dcgm-exporter-3.1.2
  name: nvdcgm-dcgm-exporter
  namespace: nvidia-dcgm
  resourceVersion: "42518"
  uid: 88963e0c-1eaa-4cb4-90ea-30561d091567
spec:
  clusterIP: 10.43.190.247
  clusterIPs:
  - 10.43.190.247
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: metrics
    port: 9400
    protocol: TCP
    targetPort: 9400
  selector:
    app.kubernetes.io/instance: nvdcgm
    app.kubernetes.io/name: dcgm-exporter
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}

kubectl port-forward service/nvdcgm-dcgm-exporter 9400:9400 -n nvidia-dcgm --address 0.0.0.0

kubectl port-forward --namespace monitoring svc/kube-stack-prometheus-grafana 3000:80 --address="0.0.0.0"