Rancher RKE2 自定义集群字段说明

本文永久链接: https://www.xtplayer.cn/rancher/rancher-rke2-custom-cluster-field-description/

apiVersion: provisioning.cattle.io/v1
kind: Cluster
metadata:
  name: rke2-test
  annotations:
    {}
    #  key: string
  labels:
    {}
    #  key: string
  namespace: fleet-default
spec:
# 自定义 cluster agent deployment 的一些配置参数，一般不需要自定义配置参数。一些特殊情况下可能需要用到，比如节点都打了 Taint，需要添加自定义 Toleration pod 才能在节点运行。
  clusterAgentDeploymentCustomization:
    appendTolerations:
#      - effect: string
#        key: string
#        operator: string
#        tolerationSeconds: int
#        value: string

    overrideAffinity:
#      nodeAffinity:
#        preferredDuringSchedulingIgnoredDuringExecution:
#          - preference:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchFields:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#            weight: int
#        requiredDuringSchedulingIgnoredDuringExecution:
#          nodeSelectorTerms:
#            - matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchFields:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#      podAffinity:
#        preferredDuringSchedulingIgnoredDuringExecution:
#          - podAffinityTerm:
#              labelSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              matchLabelKeys:
#                - string
#              mismatchLabelKeys:
#                - string
#              namespaceSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              namespaces:
#                - string
#              topologyKey: string
#            weight: int
#        requiredDuringSchedulingIgnoredDuringExecution:
#          - labelSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            matchLabelKeys:
#              - string
#            mismatchLabelKeys:
#              - string
#            namespaceSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            namespaces:
#              - string
#            topologyKey: string
#      podAntiAffinity:
#        preferredDuringSchedulingIgnoredDuringExecution:
#          - podAffinityTerm:
#              labelSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              matchLabelKeys:
#                - string
#              mismatchLabelKeys:
#                - string
#              namespaceSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              namespaces:
#                - string
#              topologyKey: string
#            weight: int
#        requiredDuringSchedulingIgnoredDuringExecution:
#          - labelSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            matchLabelKeys:
#              - string
#            mismatchLabelKeys:
#              - string
#            namespaceSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            namespaces:
#              - string
#            topologyKey: string
    overrideResourceRequirements:
#      limits:
#      requests:
# 此配置参考文档 https://ranchermanager.docs.rancher.com/zh/how-to-guides/advanced-user-guides/enable-cluster-agent-scheduling-customization。大概意思就是给 cluster agent pod 自定义 pod 优先级.
# 全局设置中启用之后，后期创建的集群默认添加以下参数。也可以编辑集群 yaml，手动添加配置。
#    schedulingCustomization:
#      podDisruptionBudget:
#        maxUnavailable: '0'
#        minAvailable: '1'
#      priorityClass:
#        preemptionPolicy: PreemptLowerPriority
#        value: 1000000000
# 参考文档 https://ranchermanager.docs.rancher.com/v2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25
# 默认空值，可选 rancher-restricted or rancher-privileged
  defaultPodSecurityAdmissionConfigurationTemplateName: ''
  # 这个跟 clusterAgentDeploymentCustomization 类似，主要是自定义 fleet agent deployment 的配置参数，一般不需要自定义。
  fleetAgentDeploymentCustomization:
    appendTolerations:
#      - effect: string
#        key: string
#        operator: string
#        tolerationSeconds: int
#        value: string
    overrideAffinity:
#      nodeAffinity:
#        preferredDuringSchedulingIgnoredDuringExecution:
#          - preference:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchFields:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#            weight: int
#        requiredDuringSchedulingIgnoredDuringExecution:
#          nodeSelectorTerms:
#            - matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchFields:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#      podAffinity:
#        preferredDuringSchedulingIgnoredDuringExecution:
#          - podAffinityTerm:
#              labelSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              matchLabelKeys:
#                - string
#              mismatchLabelKeys:
#                - string
#              namespaceSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              namespaces:
#                - string
#              topologyKey: string
#            weight: int
#        requiredDuringSchedulingIgnoredDuringExecution:
#          - labelSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            matchLabelKeys:
#              - string
#            mismatchLabelKeys:
#              - string
#            namespaceSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            namespaces:
#              - string
#            topologyKey: string
#      podAntiAffinity:
#        preferredDuringSchedulingIgnoredDuringExecution:
#          - podAffinityTerm:
#              labelSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              matchLabelKeys:
#                - string
#              mismatchLabelKeys:
#                - string
#              namespaceSelector:
#                matchExpressions:
#                  - key: string
#                    operator: string
#                    values:
#                      - string
#                matchLabels:  key: string
#              namespaces:
#                - string
#              topologyKey: string
#            weight: int
#        requiredDuringSchedulingIgnoredDuringExecution:
#          - labelSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            matchLabelKeys:
#              - string
#            mismatchLabelKeys:
#              - string
#            namespaceSelector:
#              matchExpressions:
#                - key: string
#                  operator: string
#                  values:
#                    - string
#              matchLabels:  key: string
#            namespaces:
#              - string
#            topologyKey: string
    overrideResourceRequirements:
#      limits:
#      requests:
#    这个跟 clusterAgentDeploymentCustomization 类似，自定义 fleet agent pod 的 pod 优先级
#    schedulingCustomization:
#      podDisruptionBudget:
#        maxUnavailable: string
#        minAvailable: string
#      priorityClass:
#        preemptionPolicy: string
#        value: int
# 定义安装的 rke2-k8s 版本
  kubernetesVersion: v1.33.4+rke2r1
# 当 rancher 不可用时，通过授权集群端点可用于直接访问 apiserver，而无需经过 cluster agent 进行代理通信。通过授权集群端点访问 apiserver，它具有与 cluster agent 代理访问 apiserver 相同的权限管理，权限由 kube-api-auth 进行控制。
# 注意：如果是在集群外部使用 http L7 代理 apiserver ip，并将域名解析到了代理服务器的 ip 上。如果 http L7 配置了自签名 ssl 证书，那么需要在以下的 caCerts 配置中添加自签名 ssl 证书对应的 CA 证书。附加原生 ca 内容，不需要 base64 加密。
# https://ranchermanager.docs.rancher.com/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters#4-authorized-cluster-endpoint
# https://ranchermanager.docs.rancher.com/reference-guides/rancher-manager-architecture/architecture-recommendations#architecture-for-an-authorized-cluster-endpoint-ace
  localClusterAuthEndpoint:
    caCerts: |
      -----BEGIN CERTIFICATE-----
      MIIDIzCCAgugAwIBAgIUEZ5uGymRishws37iUicJ30kW42UwDQYJKoZIhvcNAQEL
      BQAwITELMAkGA1UEBhMCQ04xEjAQBgNVBAMMCWNhdHRsZS1jYTAeFw0yNTA1MjEw
      NjQwNTlaFw0zNTA1MTkwNjQwNTlaMCExCzAJBgNVBAYTAkNOMRIwEAYDVQQDDAlj
      YXR0bGUtY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDjkjqBVawc
      zgWyvXukeRgu1ow/dYKn2g0MPEQ2Kq5DWXPN709bcC9H4TN1/VcaPMWmQZ+8rf76
      .
      .
      .
      1v/QckdvvhL/RdBxKv1KN4/nM0RIrV6vBGYvUfbkw4RqoTZ+BVQ2j8uYPNFI5tlw
      vG6CdMJdmVmzvMdguUrnImvTvGj+5j+2GZH6oSKXLq446lCTVFIV2JIOJd6//s1Y
      FDy63a0c1wGpKvQwlTHnvZgfKhbyvTd7QhhKs0q1nO3Z3OZeMS/uik3kqUSYRS7z
      bVFrPKtLuL6dfYMx7AwXkH/7eFfLR/1yuvWbX7naIyR1hAu0r46z
      -----END CERTIFICATE-----
    enabled: true
    fqdn: www.tets.com
  rkeConfig:
  # 自定义附加组件 helm 安装的 value 值。比如如果选择的网络驱动为 calico，需要自定义 calico 的运行配置。可以在这里添加自定义的 helm value 值。如下示例，自定义了 calicoctl 的镜像名称。
  # rke2 Helm Integration 说明，https://docs.rke2.io/helm 。
  # 这里可以查看 rke2 内置的附加组件 chart，https://github.com/rancher/rke2-charts/tree/main/charts。
    chartValues:
      rke2-calico:
        calicoctl:
          image: rancher/mirrored-calico-ctl-test
   # 自定义存储目录。注意：此配置只能在集群创建时修改，集群创建好之后不可以修改。
   # 非必要情况下不建议修改数据存储目录。
    dataDirectories:
      k8sDistro: '' # /x/x/rancher/rke2
      provisioning: '' # /x/x/rancher/provisioning
      systemAgent: '' # /x/x/rancher/agent
  # 定义 etcd 备份功能，默认启用
    etcd:
      disableSnapshots: false
      snapshotRetention: 5
      snapshotScheduleCron: 0 */5 * * *
      s3:
        bucket: a
        # 在 rancher ui 配置时，可以填写 accessKey 和 secretKey 后自动创建 secret。如果是 yaml 配置集群，需要在 local 集群的 cattle-global-data 命名空间中创建密文。此处定义为 <namespace-name>:<secret-name>
        cloudCredentialName: cattle-global-data:cc-6klpr
        endpoint: s3.test.com # 不加 https 协议头，不支持 http
        folder: b #(可选)
        region: c #(可选)
        skipSSLVerify: true # 如果不跳过 ca 校验，需要在 endpointCA 添加 ca 证书
        endpointCA: ''

# s3 cloudCredential 示例
#apiVersion: v1
#data:
#  s3credentialConfig-accessKey: YXNkYXNk (base64 加密)
#  s3credentialConfig-secretKey: YXNkYXNk (base64 加密)
#kind: Secret
#metadata:
#  annotations:
#  generateName: cc-
#  labels:
#    cattle.io/creator: norman
#  name: cc-6klpr
#  namespace: cattle-global-data
#type: Opaque

# 可以在全局配置中定义 rke2 的所有参数，此处定义的参数会作用于 rke2 所有节点。也可以通过下面的 machineSelectorConfig 进行选择性匹配
# rke2 完整参数参考文档:
# https://docs.rke2.io/reference/server_config
# https://docs.rke2.io/reference/linux_agent_config
    machineGlobalConfig:
      cluster-cidr: 10.42.0.0/16
      service-cidr: 10.43.0.0/16
      service-node-port-range: "30000-32767"
      cluster-dns: 10.43.0.10
      cluster-domain: "cluster.local"
      machinePoolDefaults:
        hostnameLengthLimit: '' # 15，默认会获取主机的 hostname 来定义 k8s 节点的资源名称，但是有的主机名称可能过长，导致一些不兼容的问题。设置此参数为 15，会自动截取主机 hostname 的前 15 个字符来定义 k8s 节点的资源名称。
      tls-san:
        - my-kubernetes-domain.com
        - another-kubernetes-domain.com
      cni: calico # CNI Plugins to deploy, one of none, calico, canal, cilium; optionally with multus as the first value to enable the multus meta-plugin
      disable-kube-proxy: false
      etcd-expose-metrics: false
      disable: '' # Do not deploy packaged components and delete any deployed components (valid items: rke2-coredns, rke2-ingress-nginx, rke2-metrics-server)
      etcd-arg:
        - a=1
      kube-apiserver-arg:
        - b=2
      kube-controller-manager-arg:
        - c=3
      kube-scheduler-arg:
        - d=4
#     kubelet-arg:
#       - max-pods=110
#     kube-proxy-arg:
#       - proxy-mode=ipvs
#       - ipvs-strict-arp=true
# 通过标签匹配节点，匹配到的节点设置不同的参数。其中的 config 可以添加多组，如果 config 中没有定义 matchLabels，那么表示匹配所有节点。
# 可以将 worker 节点分为多组，比如根据节点资源规格的不同，限制上面运行不同的 pod 数量。
    machineSelectorConfig:
      - config:
          protect-kernel-defaults: false
          kubelet-arg:
            - max-pods=100
#          kube-proxy-arg:
#            - proxy-mode=ipvs
#            - ipvs-strict-arp=true
      - machineLabelSelector:
          matchLabels:
            a: b
        config:
          kubelet-arg:
            - max-pods=110
      - machineLabelSelector:
          matchLabels:
            c: d
        config:
          kubelet-arg:
            - max-pods=120
#          kube-proxy-arg:
#            - proxy-mode=ipvs
#           - ipvs-strict-arp=true

# 此处对应 rke2 的 Private Registry Configuration
# 参考文档：https://docs.rke2.io/install/private_registry
    registries:
      configs:
        {}
        #authConfigSecretName: string
#        caBundle: string
#        insecureSkipVerify: boolean
#        tlsSecretName: string
      mirrors:
        {}
        #  endpoint:
#          - string
#        rewrite:  key: string
# 这个是定义更新集群时的执行策略
    upgradeStrategy:
  # 并发操作的 master 节点数，默认 1 个节点
      controlPlaneConcurrency: '1'
  # 更新 master 节点时是否驱逐，默认不驱逐。
      controlPlaneDrainOptions:
        deleteEmptyDirData: true
        disableEviction: false
        enabled: false
        force: false
        gracePeriod: -1
        ignoreDaemonSets: true
        skipWaitForDeleteTimeoutSeconds: 0
        timeout: 120
#        ignoreErrors: boolean
#        postDrainHooks:
#          - annotation: string
#        preDrainHooks:
#          - annotation: string
  # 并发操作的 worker 节点数，默认 1 个节点
      workerConcurrency: '1'
  # 更新 worker 节点时是否驱逐，默认不驱逐。
      workerDrainOptions:
        deleteEmptyDirData: true
        disableEviction: false
        enabled: false
        force: false
        gracePeriod: -1
        ignoreDaemonSets: true
        skipWaitForDeleteTimeoutSeconds: 0
        timeout: 120
#        ignoreErrors: boolean
#        postDrainHooks:
#          - annotation: string
#        preDrainHooks:
#          - annotation: string
# 集群创建好之后 apply 的 yaml 配置文件，比如要在集群创建好之后自动部署某个 deployment,可以在此定义完整的 deployment yaml。
    additionalManifest: |-
      apiVersion: monitoring.coreos.com/v1
      kind: ServiceMonitor
      metadata:
        name: rke2-${rke2_cluster_name}-metrics-export
        namespace: cattle-monitoring-system
        labels:
          rke2-cluster: rke2-${rke2_cluster_name}
      spec:
        selector:
          matchLabels:
            rke2-cluster: rke2-${rke2_cluster_name}
        endpoints:
        - port: metrics                # 对应 Service 中 ports.name
          interval: 30s
          path: /metrics
          # 如果需要 TLS 或 basic auth，在这里配置
          scheme: https
# 自定义 containerd 配置模板，与 machineSelectorConfig 类似，不同的节点可以使用不同的模板。
# 需要先将模版保存到每个节点的指定路径下，具体配置请参考以下文档。
# 配置参考文档：https://docs.rke2.io/advanced?_highlight=containerd#configuring-containerd
#    containerdSelectorConfig:
#      - containerdConfigTemplate: string
#        machineLabelSelector:
#          matchExpressions:
#            - key: string
#              operator: string
#              values:
#                - string
#          matchLabels:  key: string
    containerdSelectorConfig:
      - machineLabelSelector:
          matchLabels:
            a: test1
        containerdConfigTemplate: a=test1
      - machineLabelSelector:
          matchLabels:
            b: test2
        containerdConfigTemplate: b=test2

# 配置 cluster agent 的环境变量，比如如果环境中需要走 http proxy 才可以访问互联网，那可以通过 agent env 给 cluster agent 添加 http proxy。
  agentEnvVars:
    - name: HTTP_PROXY
      value: http://${ proxy_host }
    - name: HTTPS_PROXY
      value: http://${ proxy_host }
    - name: NO_PROXY
      value: 127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16
#   - name: string
#     value: string
# 启用网络策略
#  enableNetworkPolicy: true or false
# rke2 集群会在主机上运行 system agent systemd 服务，当 system agent 配置丢失或者配置错误的情况下，可以通过修改此参数去让 rancher 强制重新部署 system agent。但是有个前提是，必须 cluster agent 可以正常连接 rancher。
# 参考代码说明 https://github.com/rancher/rancher/blob/6886e3d097a2/pkg/apis/provisioning.cattle.io/v1/cluster_types.go#L92
#  redeploySystemAgentGeneration: int