Description:
The MySQL Operator doesn't come with any support of k8s NetworkPolicy.
If your cluster has a default network policy that denies all traffic, the operator will not be able to connect to the nodes, nor the backup job pods or router.
If I create my own network policies, and deploy them using helm, with mysql-innodbcluster as subchart, i can make it work, but if I uninstall the helm chart, it is likely that the network policies are removed before the operatore removes the cluster, resulting in that the operator cannot reach the cluster any more that it needs to remove. This can only be properly solved by adding the operator as finalizer of the network policies.
How to repeat:
deploy a default network policy like:
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
annotations:
k8s.portavita.net/hint: deny-all
name: default-network-policy
spec:
egress:
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
to:
- namespaceSelector:
matchLabels:
name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
podSelector: {}
policyTypes:
- Ingress
- Egress
then deploy a mysql-innodbcluster and operator
Suggested fix:
optionally include network policies in the helm charts.
The network policies need the operator as finalizer, so they are deleted after the cluster has been deleted.
the backup jobs will need a default label to be able to match them
netpol for the operator:
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ .Release.Name }}
labels:
release: {{ .Release.Name }}
spec:
policyTypes:
- Egress
egress:
- to:
{{- range .Values.netpol.ApiServerIps }}
- ipBlock:
cidr: {{ . }}/32
{{- end }}
- namespaceSelector:
matchLabels:
name: kube-system
podSelector:
matchLabels:
component: kube-apiserver
tier: control-plane
ports:
- port: {{ .Values.netpol.apiPort }}
- to:
- namespaceSelector: { }
podSelector:
matchLabels:
app.kubernetes.io/component: database
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
ports:
- port: 3306 #From MySQL Shell to the MySQL server (classic MySQL protocol).
protocol: TCP
- port: 33060 #From MySQL Shell to the MySQL server (X Protocol)
protocol: TCP
podSelector:
matchLabels:
name: mysql-operator
netpol cluster:
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ .Release.Name }}-cluster
labels:
release: {{ .Release.Name }}
spec:
policyTypes:
- Egress
- Ingress
egress:
- to: #apiserver
{{- range .Values.netpol.ApiServerIps }}
- ipBlock:
cidr: {{ . }}/32
{{- end }}
- namespaceSelector:
matchLabels:
name: kube-system
podSelector:
matchLabels:
component: kube-apiserver
tier: control-plane
ports:
- port: {{ .Values.netpol.apiPort }}
- to: #peers
- podSelector:
matchLabels:
app.kubernetes.io/component: database
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/name: mysql-innodbcluster-mysql-server
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server
ports:
- port: 3306
protocol: TCP
ingress:
- from: #operator
- namespaceSelector:
matchLabels:
name: kube-system
podSelector:
matchLabels:
name: mysql-operator
ports:
- port: 3306
protocol: TCP
- port: 33060
protocol: TCP
- from: #router
- podSelector:
matchLabels:
component: mysqlrouter
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-router
ports:
- port: 3306
protocol: TCP
- port: 33060
protocol: TCP
- from: #peers
- podSelector:
matchLabels:
app.kubernetes.io/component: database
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/name: mysql-innodbcluster-mysql-server
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server
ports:
- port: 3306
protocol: TCP
- from: #backup snapshot jobs
- podSelector:
matchLabels:
backupjob: {{ .Release.Name }}
ports:
- port: 3306
protocol: TCP
podSelector:
matchLabels:
app.kubernetes.io/component: database
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/name: mysql-innodbcluster-mysql-server
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server
(and add stuff for loading backups too?)
for the router:
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ .Release.Name }}-router
labels:
release: {{ .Release.Name }}
spec:
policyTypes:
- Egress
- Ingress
egress:
- to: #cluster
- podSelector:
matchLabels:
app.kubernetes.io/component: database
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/name: mysql-innodbcluster-mysql-server
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server
ports:
- port: 3306
protocol: TCP
- port: 33060
protocol: TCP
- port: 33061
protocol: TCP
ingress:
- from: #clients
- podSelector:
matchLabels:
{{ .Release.Name }}-client: "true"
ports:
- port: 6446 #RW Classic
protocol: TCP
- port: 6447 #R Classic
protocol: TCP
- port: 6448 #RW X
protocol: TCP
- port: 6449 #R X
protocol: TCP
- port: 6450 #RW split connections Classic
protocol: TCP
podSelector:
matchLabels:
app.kubernetes.io/component: router
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/name: mysql-router
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-router
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
{{- end }}
metadata:
name: {{ .Release.Name }}-backupjob
labels:
release: {{ .Release.Name }}
spec:
policyTypes:
- Egress
egress:
- to: #apiserver
{{- range .Values.netpol.ApiServerIps }}
- ipBlock:
cidr: {{ . }}/32
{{- end }}
- namespaceSelector:
matchLabels:
name: kube-system
podSelector:
matchLabels:
component: kube-apiserver
tier: control-plane
ports:
- port: {{ .Values.netpol.apiPort }}
- to: #cluster
- podSelector:
matchLabels:
app.kubernetes.io/component: database
app.kubernetes.io/created-by: mysql-operator
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/name: mysql-innodbcluster-mysql-server
app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server
ports:
- port: 3306
protocol: TCP
- port: 33060
protocol: TCP
- port: 33061
protocol: TCP
podSelector:
matchLabels:
backupjob: {{ .Release.Name }}
+logic for fetching the backups from s3, oci, ...
Description: The MySQL Operator doesn't come with any support of k8s NetworkPolicy. If your cluster has a default network policy that denies all traffic, the operator will not be able to connect to the nodes, nor the backup job pods or router. If I create my own network policies, and deploy them using helm, with mysql-innodbcluster as subchart, i can make it work, but if I uninstall the helm chart, it is likely that the network policies are removed before the operatore removes the cluster, resulting in that the operator cannot reach the cluster any more that it needs to remove. This can only be properly solved by adding the operator as finalizer of the network policies. How to repeat: deploy a default network policy like: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: annotations: k8s.portavita.net/hint: deny-all name: default-network-policy spec: egress: - ports: - port: 53 protocol: UDP - port: 53 protocol: TCP to: - namespaceSelector: matchLabels: name: kube-system podSelector: matchLabels: k8s-app: kube-dns podSelector: {} policyTypes: - Ingress - Egress then deploy a mysql-innodbcluster and operator Suggested fix: optionally include network policies in the helm charts. The network policies need the operator as finalizer, so they are deleted after the cluster has been deleted. the backup jobs will need a default label to be able to match them netpol for the operator: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ .Release.Name }} labels: release: {{ .Release.Name }} spec: policyTypes: - Egress egress: - to: {{- range .Values.netpol.ApiServerIps }} - ipBlock: cidr: {{ . }}/32 {{- end }} - namespaceSelector: matchLabels: name: kube-system podSelector: matchLabels: component: kube-apiserver tier: control-plane ports: - port: {{ .Values.netpol.apiPort }} - to: - namespaceSelector: { } podSelector: matchLabels: app.kubernetes.io/component: database app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator ports: - port: 3306 #From MySQL Shell to the MySQL server (classic MySQL protocol). protocol: TCP - port: 33060 #From MySQL Shell to the MySQL server (X Protocol) protocol: TCP podSelector: matchLabels: name: mysql-operator netpol cluster: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ .Release.Name }}-cluster labels: release: {{ .Release.Name }} spec: policyTypes: - Egress - Ingress egress: - to: #apiserver {{- range .Values.netpol.ApiServerIps }} - ipBlock: cidr: {{ . }}/32 {{- end }} - namespaceSelector: matchLabels: name: kube-system podSelector: matchLabels: component: kube-apiserver tier: control-plane ports: - port: {{ .Values.netpol.apiPort }} - to: #peers - podSelector: matchLabels: app.kubernetes.io/component: database app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator app.kubernetes.io/name: mysql-innodbcluster-mysql-server app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server ports: - port: 3306 protocol: TCP ingress: - from: #operator - namespaceSelector: matchLabels: name: kube-system podSelector: matchLabels: name: mysql-operator ports: - port: 3306 protocol: TCP - port: 33060 protocol: TCP - from: #router - podSelector: matchLabels: component: mysqlrouter app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-router ports: - port: 3306 protocol: TCP - port: 33060 protocol: TCP - from: #peers - podSelector: matchLabels: app.kubernetes.io/component: database app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator app.kubernetes.io/name: mysql-innodbcluster-mysql-server app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server ports: - port: 3306 protocol: TCP - from: #backup snapshot jobs - podSelector: matchLabels: backupjob: {{ .Release.Name }} ports: - port: 3306 protocol: TCP podSelector: matchLabels: app.kubernetes.io/component: database app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator app.kubernetes.io/name: mysql-innodbcluster-mysql-server app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server (and add stuff for loading backups too?) for the router: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ .Release.Name }}-router labels: release: {{ .Release.Name }} spec: policyTypes: - Egress - Ingress egress: - to: #cluster - podSelector: matchLabels: app.kubernetes.io/component: database app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator app.kubernetes.io/name: mysql-innodbcluster-mysql-server app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server ports: - port: 3306 protocol: TCP - port: 33060 protocol: TCP - port: 33061 protocol: TCP ingress: - from: #clients - podSelector: matchLabels: {{ .Release.Name }}-client: "true" ports: - port: 6446 #RW Classic protocol: TCP - port: 6447 #R Classic protocol: TCP - port: 6448 #RW X protocol: TCP - port: 6449 #R X protocol: TCP - port: 6450 #RW split connections Classic protocol: TCP podSelector: matchLabels: app.kubernetes.io/component: router app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator app.kubernetes.io/name: mysql-router app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-router apiVersion: networking.k8s.io/v1 kind: NetworkPolicy {{- end }} metadata: name: {{ .Release.Name }}-backupjob labels: release: {{ .Release.Name }} spec: policyTypes: - Egress egress: - to: #apiserver {{- range .Values.netpol.ApiServerIps }} - ipBlock: cidr: {{ . }}/32 {{- end }} - namespaceSelector: matchLabels: name: kube-system podSelector: matchLabels: component: kube-apiserver tier: control-plane ports: - port: {{ .Values.netpol.apiPort }} - to: #cluster - podSelector: matchLabels: app.kubernetes.io/component: database app.kubernetes.io/created-by: mysql-operator app.kubernetes.io/managed-by: mysql-operator app.kubernetes.io/name: mysql-innodbcluster-mysql-server app.kubernetes.io/instance: mysql-innodbcluster-{{ .Release.Name }}-mysql-server ports: - port: 3306 protocol: TCP - port: 33060 protocol: TCP - port: 33061 protocol: TCP podSelector: matchLabels: backupjob: {{ .Release.Name }} +logic for fetching the backups from s3, oci, ...