检查rdma网卡

启动multus

拷贝macvlan cni文件

从https://github.com/containernetworking/plugins/release下载后拷贝到/opt/cni/bin/

netAttatch文件

apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: macvlan-net1
spec:
  config: '{
  "cniVersion": "0.3.1",
  "type": "macvlan",
  "master": "enp61s0np0",
  "mode": "bridge",
  "ipam": {
    "type": "whereabouts",
    "range": "192.168.107.0/27",
    "range_start": "192.168.107.2",
    "range_end": "192.168.107.10",
    "routes": [
         { "dst": "192.168.0.0/16" },
         { "dst": "192.168.107.32/27" }
    ],
    "gateway": "192.168.107.1" 
  }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: macvlan-net2
spec:
  config: '{
  "cniVersion": "0.3.1",
  "type": "macvlan",
  "master": "ens10np0",
  "mode": "bridge",
  "ipam": {
    "type": "whereabouts",
    "range": "192.168.107.32/27",
    "range_start": "192.168.107.34",
    "range_end": "192.168.107.63",
    "routes": [
         { "dst": "192.168.0.0/16" },
         { "dst": "192.168.107.0/27" }
    ],
    "gateway": "192.168.107.33"
  }
}'

pod文件

apiVersion: v1
kind: Pod
metadata:
  name: pod1
  annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-net1,macvlan-net2
spec:
  nodeName: wsgpu02
  restartPolicy: OnFailure
  initContainers:
  - image: busybox
    command:
    - sh
    - -c
    - |
      sysctl -w net.ipv4.conf.all.arp_announce=2
      sysctl -w net.ipv4.conf.all.arp_filter=0
      sysctl -w net.ipv4.conf.all.rp_filter=2
      sysctl -w net.ipv4.conf.all.accept_local=1
      sysctl -w net.ipv4.conf.all.arp_ignore=1
    imagePullPolicy: IfNotPresent
    name: setsysctl
    securityContext:
      privileged: true
  containers:
  - image: docker.io/library/nccl-tests:24.12
    imagePullPolicy: IfNotPresent
    name: nccl-test
    securityContext:
      capabilities:
        add: [ "IPC_LOCK" ]
    #resources:
      #limits:       
        #nvidia.com/gpu: 4
        #requests:       
        #nvidia.com/gpu: 4
    command:
    - sh
    - -c
    - |
      ls -l /dev/infiniband /sys/class/infiniband /sys/class/net
      sleep 1000000