MySQL Bugs: #108010: podspec can not support nodeAffinity

Bug #108010	podspec can not support nodeAffinity
Submitted:	28 Jul 2022 6:07	Modified:	5 Oct 2022 12:40
Reporter:	wu wu	Email Updates:
Status:	Won't fix	Impact on me:	None
Category:	MySQL Operator	Severity:	S3 (Non-critical)
Version:	8.0.30-2.0.5	OS:	Linux
Assigned to:		CPU Architecture:	Any

Description:
kubernetes version  v1.22.10

mysql operator version 8.0.30-2.0.5
 
mysql server version 8.0.29

I want use nodeAffinity to choice designative node install mysql. but not work.

statefulset not have nodeAffinity  

How to repeat:
innodbcluster yaml:
```yaml
apiVersion: mysql.oracle.com/v2
kind: InnoDBCluster
metadata:
  name: mycluster
  namespace: mysql-operator
spec:
  secretName: mypwds
  tlsUseSelfSigned: true
  
  instances: 1
  router:
    instances: 1
  version: "8.0.29"
  podspec:
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: mysql
                  operator: In
                  values:
                    - mysql
```
statefulset yaml

```yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app.kubernetes.io/component: database
    app.kubernetes.io/created-by: mysql-operator
    app.kubernetes.io/instance: mysql-innodbcluster-mycluster
    app.kubernetes.io/managed-by: mysql-operator
    app.kubernetes.io/name: mysql-innodbcluster
    mysql.oracle.com/cluster: mycluster
    objectset.rio.cattle.io/hash: 7cae79ba428554921e09b68e5ba1724c94818c3d
    tier: mysql
  name: mycluster
  namespace: mysql-operator
  ownerReferences:
    - apiVersion: mysql.oracle.com/v2
      blockOwnerDeletion: true
      controller: true
      kind: InnoDBCluster
      name: mycluster
      uid: 1b54849f-ce9c-43f0-aef2-37501ca8dd19
spec:
  podManagementPolicy: Parallel
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/component: database
      app.kubernetes.io/created-by: mysql-operator
      app.kubernetes.io/instance: mysql-innodbcluster-mycluster-mysql-server
      app.kubernetes.io/managed-by: mysql-operator
      app.kubernetes.io/name: mysql-innodbcluster-mysql-server
      component: mysqld
      mysql.oracle.com/cluster: mycluster
      tier: mysql
  serviceName: mycluster-instances
  template:
    metadata:
      creationTimestamp: null
      labels:
        app.kubernetes.io/component: database
        app.kubernetes.io/created-by: mysql-operator
        app.kubernetes.io/instance: mysql-innodbcluster-mycluster-mysql-server
        app.kubernetes.io/managed-by: mysql-operator
        app.kubernetes.io/name: mysql-innodbcluster-mysql-server
        component: mysqld
        mysql.oracle.com/cluster: mycluster
        tier: mysql
    spec:
      containers:
        - command:
            - mysqlsh
            - --pym
            - mysqloperator
            - sidecar
          env:
            - name: MY_POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: MY_POD_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: MYSQL_UNIX_PORT
              value: /var/run/mysqld/mysql.sock
            - name: MYSQLSH_USER_CONFIG_HOME
              value: /mysqlsh
          image: mysql/mysql-operator:8.0.30-2.0.5
          imagePullPolicy: Always
          name: sidecar
          resources: {}
          securityContext:
            runAsUser: 27
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /var/run/mysqld
              name: rundir
            - mountPath: /etc/my.cnf.d
              name: mycnfdata
              subPath: my.cnf.d
            - mountPath: /etc/my.cnf
              name: mycnfdata
              subPath: my.cnf
            - mountPath: /mysqlsh
              name: shellhome
        - args:
            - mysqld
            - --user=mysql
          env:
            - name: MYSQL_UNIX_PORT
              value: /var/run/mysqld/mysql.sock
          image: mysql/mysql-server:8.0.29
          imagePullPolicy: Always
          lifecycle:
            preStop:
              exec:
                command:
                  - sh
                  - -c
                  - sleep 20 && mysqladmin -ulocalroot shutdown
          livenessProbe:
            exec:
              command:
                - /livenessprobe.sh
            failureThreshold: 10
            initialDelaySeconds: 15
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 1
          name: mysql
          ports:
            - containerPort: 3306
              name: mysql
              protocol: TCP
            - containerPort: 33060
              name: mysqlx
              protocol: TCP
            - containerPort: 33061
              name: gr-xcom
              protocol: TCP
          readinessProbe:
            exec:
              command:
                - /readinessprobe.sh
            failureThreshold: 10000
            initialDelaySeconds: 10
            periodSeconds: 5
            successThreshold: 1
            timeoutSeconds: 1
          resources: {}
          securityContext:
            capabilities:
              add:
                - DAC_OVERRIDE
                - SETGID
                - SETUID
                - SYS_NICE
                - SYS_RESOURCE
              drop:
                - AUDIT_CONTROL
                - AUDIT_WRITE
                - BLOCK_SUSPEND
                - CHOWN
                - DAC_READ_SEARCH
                - FOWNER
                - FSETID
                - IPC_LOCK
                - IPC_OWNER
                - KILL
                - LEASE
                - LINUX_IMMUTABLE
                - MAC_ADMIN
                - MAC_OVERRIDE
                - MKNOD
                - NET_ADMIN
                - NET_BIND_SERVICE
                - NET_BROADCAST
                - NET_RAW
                - SETFCAP
                - SETPCAP
                - SYS_ADMIN
                - SYS_BOOT
                - SYS_CHROOT
                - SYS_MODULE
                - SYS_PACCT
                - SYS_PTRACE
                - SYS_RAWIO
                - SYS_TIME
                - SYS_TTY_CONFIG
                - SYSLOG
                - WAKE_ALARM
          startupProbe:
            exec:
              command:
                - /livenessprobe.sh
                - "8"
            failureThreshold: 10000
            initialDelaySeconds: 5
            periodSeconds: 3
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /var/lib/mysql
              name: datadir
            - mountPath: /var/run/mysqld
              name: rundir
            - mountPath: /etc/my.cnf.d
              name: mycnfdata
              subPath: my.cnf.d
            - mountPath: /etc/my.cnf
              name: mycnfdata
              subPath: my.cnf
            - mountPath: /livenessprobe.sh
              name: initconfdir
              subPath: livenessprobe.sh
            - mountPath: /readinessprobe.sh
              name: initconfdir
              subPath: readinessprobe.sh
      dnsPolicy: ClusterFirst
      initContainers:
        - command:
            - bash
            - -c
            - chown 27:27 /var/lib/mysql && chmod 0700 /var/lib/mysql
          image: mysql/mysql-operator:8.0.30-2.0.5
          imagePullPolicy: Always
          name: fixdatadir
          resources: {}
          securityContext:
            runAsUser: 0
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /var/lib/mysql
              name: datadir
        - command:
            - mysqlsh
            - --log-level=@INFO
            - --pym
            - mysqloperator
            - init
          env:
            - name: MY_POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: MY_POD_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: MYSQLSH_USER_CONFIG_HOME
              value: /tmp
          image: mysql/mysql-operator:8.0.30-2.0.5
          imagePullPolicy: Always
          name: initconf
          resources: {}
          securityContext:
            runAsUser: 27
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /mnt/initconf
              name: initconfdir
              readOnly: true
            - mountPath: /var/lib/mysql
              name: datadir
            - mountPath: /mnt/mycnfdata
              name: mycnfdata
        - args:
            - mysqld
            - --user=mysql
          env:
            - name: MYSQL_INITIALIZE_ONLY
              value: "1"
            - name: MYSQL_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: rootPassword
                  name: mypwds
            - name: MYSQLSH_USER_CONFIG_HOME
              value: /tmp
          image: mysql/mysql-server:8.0.29
          imagePullPolicy: Always
          name: initmysql
          resources: {}
          securityContext:
            capabilities:
              add:
                - DAC_OVERRIDE
                - SETGID
                - SETUID
                - SYS_NICE
                - SYS_RESOURCE
              drop:
                - AUDIT_CONTROL
                - AUDIT_WRITE
                - BLOCK_SUSPEND
                - CHOWN
                - DAC_READ_SEARCH
                - FOWNER
                - FSETID
                - IPC_LOCK
                - IPC_OWNER
                - KILL
                - LEASE
                - LINUX_IMMUTABLE
                - MAC_ADMIN
                - MAC_OVERRIDE
                - MKNOD
                - NET_ADMIN
                - NET_BIND_SERVICE
                - NET_BROADCAST
                - NET_RAW
                - SETFCAP
                - SETPCAP
                - SYS_ADMIN
                - SYS_BOOT
                - SYS_CHROOT
                - SYS_MODULE
                - SYS_PACCT
                - SYS_PTRACE
                - SYS_RAWIO
                - SYS_TIME
                - SYS_TTY_CONFIG
                - SYSLOG
                - WAKE_ALARM
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /var/lib/mysql
              name: datadir
            - mountPath: /var/run/mysqld
              name: rundir
            - mountPath: /etc/my.cnf.d
              name: mycnfdata
              subPath: my.cnf.d
            - mountPath: /docker-entrypoint-initdb.d
              name: mycnfdata
              subPath: docker-entrypoint-initdb.d
            - mountPath: /etc/my.cnf
              name: mycnfdata
              subPath: my.cnf
      readinessGates:
        - conditionType: mysql.oracle.com/configured
        - conditionType: mysql.oracle.com/ready
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 27
        runAsGroup: 27
        runAsUser: 27
      serviceAccount: mycluster-sidecar-sa
      serviceAccountName: mycluster-sidecar-sa
      subdomain: mycluster
      terminationGracePeriodSeconds: 30
      volumes:
        - emptyDir: {}
          name: mycnfdata
        - emptyDir: {}
          name: rundir
        - configMap:
            defaultMode: 493
            name: mycluster-initconf
          name: initconfdir
        - emptyDir: {}
          name: shellhome
  updateStrategy:
    rollingUpdate:
      partition: 0
    type: RollingUpdate
  volumeClaimTemplates:
    - apiVersion: v1
      kind: PersistentVolumeClaim
      metadata:
        name: datadir
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: 2Gi
        volumeMode: Filesystem
      status:
        phase: Pending
```

```

Hi,

Thanks for the report

Kind regards

I have similar issue,

seems everything under podSpec get's ignored, including resources & affinity.

The repository also lacks the examples including podSpec

Posted by developer:
 
I can not reproduce the problem. For me it works well:

Given this spec:

apiVersion: mysql.oracle.com/v2
kind: InnoDBCluster
metadata:
  name: myaffinity

spec:
  instances: 3
  secretName: mypwds

  podSpec:
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: mysql
                  operator: In
                  values:
                    - mysql

  router: 
    instances: 1
    podSpec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: mysqlrouter
                    operator: In
                    values:
                      - mysqlrouter

I apply it and then check the created sts and deployment:

$ kubectl get statefulsets myaffinity -o json | jq .spec.template.spec.affinity
{
  "nodeAffinity": {
    "requiredDuringSchedulingIgnoredDuringExecution": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "mysql",
              "operator": "In",
              "values": [
                "mysql"
              ]
            }
          ]
        }
      ]
    }
  }
}

$ kubectl get deployment myaffinity-router -o json | jq .spec.template.spec.affinity 
{
  "nodeAffinity": {
    "requiredDuringSchedulingIgnoredDuringExecution": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "mysqlrouter",
              "operator": "In",
              "values": [
                "mysqlrouter"
              ]
            }
          ]
        }
      ]
    }
  }
}

Posted by developer:
 
With your manifest I get the expected error:

error: error validating "...": error validating data: ValidationError(InnoDBCluster.spec): unknown field "podspec" in com.oracle.mysql.v2.InnoDBCluster.spec; if you choose to ignore these errors, turn validation off with --validate=false

when changing "podspec" to "podSpec" (capital S) the affinity rules are created.