最近花了几天时间研究在k8s集群中部署rabbitmq，记录经过。
前提：
1.k8s集群(版本 1.16.15)
2.存储（由于rabbitmq等中间件是有状态的需要存储数据，所以需要提供状态存储，这里我使用的是longhorn，当然也可以使用nfs，部署longhorn空余时间写一下教程）
3.nginx ingress：这里需要使用nginx ingress暴露 rabbitmq web管理页面以及5672的rabbitmq amqp tcp协议端口
4.rabbitmq docker镜像（这里因为我需要开启rabbitmq_delayed_message_exchange插件，但是官方镜像不包含，所以基于官方镜像做了修改）
rabbitmq-cluster.yaml
---
apiVersion: v1
kind: Namespace
metadata:
name: ops
---
apiVersion: v1
kind: ConfigMap
metadata:
name: rabbitmq-config
namespace: ops
data:
enabled_plugins: |
[rabbitmq_management,rabbitmq_peer_discovery_k8s,rabbitmq_delayed_message_exchange].
rabbitmq.conf: |
## Cluster formation. See https://www.rabbitmq.com/cluster-formation.html to learn more.
cluster_formation.randomized_startup_delay_range.min = 0
cluster_formation.randomized_startup_delay_range.max = 1
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
## Service name is rabbitmq by default but can be overridden using the cluster_formation.k8s.service_name key if needed
cluster_formation.k8s.service_name = rabbitmq
## It is possible to append a suffix to peer hostnames returned by Kubernetes using cluster_formation.k8s.hostname_suffix
cluster_formation.k8s.hostname_suffix = .rabbitmq.ops.svc.cluster.local
## Should RabbitMQ node name be computed from the pod's hostname or IP address?
## IP addresses are not stable, so using [stable] hostnames is recommended when possible.
## Set to "hostname" to use pod hostnames.
## When this value is changed, so should the variable used to set the RABBITMQ_NODENAME
## environment variable.
cluster_formation.k8s.address_type = hostname
## How often should node cleanup checks run?
cluster_formation.node_cleanup.interval = 30
## Set to false if automatic removal of unknown/absent nodes
## is desired. This can be dangerous, see
## * https://www.rabbitmq.com/cluster-formation.html#node-health-checks-and-cleanup
## * https://groups.google.com/forum/#!msg/rabbitmq-users/wuOfzEywHXo/k8z_HWIkBgAJ
cluster_formation.node_cleanup.only_log_warning = true
cluster_partition_handling = autoheal
## See https://www.rabbitmq.com/ha.html#master-migration-data-locality
queue_master_locator=min-masters
## This is just an example.
## This enables remote access for the default user with well known credentials.
## Consider deleting the default user and creating a separate user with a set of generated
## credentials instead.
## Learn more at https://www.rabbitmq.com/access-control.html#loopback-users
loopback_users.guest = false
## https://www.rabbitmq.com/memory.html#configuring-threshold
vm_memory_high_watermark.relative = 0.6
---
apiVersion: v1
kind: Secret
metadata:
name: rabbitmq-secret
namespace: ops
type: Opaque
data:
RABBITMQ_ERLANG_COOKIE: MTIzNDU=
RABBITMQ_DEFAULT_USER: YWRtaW4=
RABBITMQ_DEFAULT_PASS: YWRtaW4=
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rabbitmq
namespace: ops
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rabbitmq-peer-discovery-rbac
namespace: ops
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rabbitmq-peer-discovery-rbac
namespace: ops
subjects:
- kind: ServiceAccount
name: rabbitmq
namespace: ops
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rabbitmq-peer-discovery-rbac
---
kind: Service
apiVersion: v1
metadata:
namespace: ops
name: rabbitmq
labels:
app: rabbitmq
spec:
ports:
- name: epmd
protocol: TCP
port: 4369
- name: amqp
protocol: TCP
port: 5672
- name: amqp-tls
protocol: TCP
port: 5671
- name: http
protocol: TCP
port: 15672
- name: inter-node-cli
protocol: TCP
port: 25672
selector:
app: rabbitmq
---
apiVersion: apps/v1
# See the Prerequisites section of https://www.rabbitmq.com/cluster-formation.html#peer-discovery-k8s.
kind: StatefulSet
metadata:
name: rabbitmq
namespace: ops
spec:
serviceName: rabbitmq
# Three nodes is the recommended minimum. Some features may require a majority of nodes
# to be available.
replicas: 3
selector:
matchLabels:
app: rabbitmq
template:
metadata:
labels:
app: rabbitmq
spec:
serviceAccountName: rabbitmq
terminationGracePeriodSeconds: 10
nodeSelector:
# Use Linux nodes in a mixed OS kubernetes cluster.
# Learn more at https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#kubernetes-io-os
kubernetes.io/os: linux
initContainers:
- name: fix-readonly-config
image: busybox:1.31.1
command:
- sh
- -c
- cp /tmp/config/* /etc/rabbitmq;
volumeMounts:
- name: rabbitmq-config
mountPath: /etc/rabbitmq
- name: tmp-dir
mountPath: /tmp/config
containers:
- name: rabbitmq
image: rabbitmq:3.8.17-management
# Learn more about what ports various protocols use
# at https://www.rabbitmq.com/networking.html#ports
ports:
- name: epmd
protocol: TCP
containerPort: 4369
- name: amqp
protocol: TCP
containerPort: 5672
- name: amqp-tls
protocol: TCP
containerPort: 5671
- name: http
protocol: TCP
containerPort: 15672
env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: RABBITMQ_NODENAME
value: rabbit@$(POD_NAME).rabbitmq.$(POD_NAMESPACE).svc.cluster.local
- name: RABBITMQ_USE_LONGNAME
value: "true"
envFrom:
- secretRef:
name: rabbitmq-secret
volumeMounts:
- name: rabbitmq-config
mountPath: /etc/rabbitmq
- name: rabbitmq-data
mountPath: /var/lib/rabbitmq
volumes:
- name: rabbitmq-config
emptyDir: {}
- name: tmp-dir
configMap:
name: rabbitmq-config
- name: rabbitmq-data
persistentVolumeClaim:
claimName: rabbitmq-data
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
description: rabbitmq
nginx.ingress.kubernetes.io/proxy-body-size: 50m
name: rabbitmq
namespace: ops
spec:
rules:
- host: rabbitmq.yours.com
http:
paths:
- backend:
serviceName: rabbitmq
servicePort: 15672
path: /
status:
loadBalancer: {}
kubectl apply -f rabbitmq-cluster.yaml