This article will show you how to provision a custom etcd cluster for service discovery on top of Kubernetes; it is unrelated with the default etcd that drives Kubernetes.
Can be used to store key/value pair and for dynamic service that will register to it, live based on TTL settings.
Article presents “part 1”, the creation of the isolated etcd cluster.
Prerequisites – environment as tested
Kubernetes: 1.8.15
Cloud provider: Amazon
Docker containers: Alpine Linux 3.8
Golang: 1.11
KubeDNS: pre-configured
Public personal repository for Docker images: zeding/*
Create all yaml templates
You will end up with 3 files in a folder called test-etcd
# service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: infra-etcd-cluster
app: etcd
name: infra-etcd-cluster
spec:
clusterIP: None
ports:
- name: infra-etcd-cluster-2379
port: 2379
protocol: TCP
targetPort: 2379
- name: infra-etcd-cluster-2380
port: 2380
protocol: TCP
targetPort: 2380
selector:
k8s-app: infra-etcd-cluster
app: etcd
type: ClusterIP
# service-client.yaml
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: etcd-client-service
app: etcd
name: etcd-client-service
spec:
ports:
- name: infra-etcd-cluster-2379
port: 2379
protocol: TCP
targetPort: 2379
selector:
k8s-app: infra-etcd-cluster
app: etcd
sessionAffinity: None
type: ClusterIP
# etcd-cluster.yaml
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
labels:
k8s-app: infra-etcd-cluster
app: etcd
name: infra-etcd-cluster
spec:
replicas: 3
selector:
matchLabels:
k8s-app: infra-etcd-cluster
app: etcd
serviceName: infra-etcd-cluster
template:
metadata:
labels:
k8s-app: infra-etcd-cluster
app: etcd
name: infra-etcd-cluster
spec:
containers:
- command:
- /bin/sh
- -ec
- |
HOSTNAME=$(hostname)
echo "etcd api version is ${ETCDAPI_VERSION}"
eps() {
EPS=""
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}.${CLUSTER_NAMESPACE}:2379"
done
echo ${EPS}
}
member_hash() {
etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380 | cut -d':' -f1 | cut -d'[' -f1
}
initial_peers() {
PEERS=""
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380"
done
echo ${PEERS}
}
# etcd-SET_ID
SET_ID=${HOSTNAME##*-}
# adding a new member to existing cluster (assuming all initial pods are available)
if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then
export ETCDCTL_ENDPOINTS=$(eps)
# member already added?
MEMBER_HASH=$(member_hash)
if [ -n "${MEMBER_HASH}" ]; then
# the member hash exists but for some reason etcd failed
# as the datadir has not be created, we can remove the member
# and retrieve new hash
if [ "${ETCDAPI_VERSION}" -eq 3 ]; then
ETCDCTL_API=3 etcdctl --user=root:${ROOT_PASSWORD} member remove ${MEMBER_HASH}
else
etcdctl --username=root:${ROOT_PASSWORD} member remove ${MEMBER_HASH}
fi
fi
echo "Adding new member"
rm -rf /var/run/etcd/*
# ensure etcd dir exist
mkdir -p /var/run/etcd/
# sleep 60s wait endpoint become ready
echo "sleep 60s wait endpoint become ready,sleeping..."
sleep 60
if [ "${ETCDAPI_VERSION}" -eq 3 ]; then
ETCDCTL_API=3 etcdctl --user=root:${ROOT_PASSWORD} member add ${HOSTNAME} --peer-urls=http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs
else
etcdctl --username=root:${ROOT_PASSWORD} member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs
fi
if [ $? -ne 0 ]; then
echo "member add ${HOSTNAME} error."
rm -f /var/run/etcd/new_member_envs
exit 1
fi
cat /var/run/etcd/new_member_envs
source /var/run/etcd/new_member_envs
exec etcd --name ${HOSTNAME} \
--initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380 \
--listen-peer-urls http://0.0.0.0:2380 \
--listen-client-urls http://0.0.0.0:2379 \
--advertise-client-urls http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2379 \
--data-dir /var/run/etcd/default.etcd \
--initial-cluster ${ETCD_INITIAL_CLUSTER} \
--initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE}
fi
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
while true; do
echo "Waiting for ${SET_NAME}-${i}.${SET_NAME}.${CLUSTER_NAMESPACE} to come up"
ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME}.${CLUSTER_NAMESPACE} > /dev/null && break
sleep 1s
done
done
echo "join member ${HOSTNAME}"
# join member
exec etcd --name ${HOSTNAME} \
--initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380 \
--listen-peer-urls http://0.0.0.0:2380 \
--listen-client-urls http://0.0.0.0:2379 \
--advertise-client-urls http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2379 \
--initial-cluster-token etcd-cluster-1 \
--data-dir /var/run/etcd/default.etcd \
--initial-cluster $(initial_peers) \
--initial-cluster-state new
env:
- name: INITIAL_CLUSTER_SIZE
value: "3"
- name: CLUSTER_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: ETCDAPI_VERSION
value: "3"
- name: ROOT_PASSWORD
value: "Password123"
- name: SET_NAME
value: "infra-etcd-cluster"
- name: GOMAXPROCS
value: "4"
image: zeding/alpine-etcd:3.3.9
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -ec
- |
HOSTNAME=$(hostname)
member_hash() {
etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}.${CLUSTER_NAMESPACE}:2380 | cut -d':' -f1 | cut -d'[' -f1
}
eps() {
EPS=""
for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}.${CLUSTER_NAMESPACE}:2379"
done
echo ${EPS}
}
export ETCDCTL_ENDPOINTS=$(eps)
SET_ID=${HOSTNAME##*-}
# Removing member from cluster
if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then
echo "Removing ${HOSTNAME} from etcd cluster"
if [ "${ETCDAPI_VERSION}" -eq 3 ]; then
ETCDCTL_API=3 etcdctl --user=root:${ROOT_PASSWORD} member remove $(member_hash)
else
etcdctl --username=root:${ROOT_PASSWORD} member remove $(member_hash)
fi
if [ $? -eq 0 ]; then
# Remove everything otherwise the cluster will no longer scale-up
rm -rf /var/run/etcd/*
fi
fi
name: infra-etcd-cluster
ports:
- containerPort: 2380
name: peer
protocol: TCP
- containerPort: 2379
name: client
protocol: TCP
resources:
limits:
cpu: "1"
memory: 1Gi
requests:
cpu: "1"
memory: 1Gi
updateStrategy:
type: OnDelete
Provisioning the new etcd-cluster
cd test-etcd
kubectl -n test-namespace create -f .
Test you new cluster
kubectl -n test-namespace get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE
infra-etcd-cluster-0 1/1 Running 0 1d 100.96.13.85 ip-172-20-42-48.eu-central-1.compute.internal
infra-etcd-cluster-1 1/1 Running 0 1d 100.96.15.148 ip-172-20-44-47.eu-central-1.compute.internal
infra-etcd-cluster-2 1/1 Running 0 1d 100.96.10.144 ip-172-20-43-86.eu-central-1.compute.internal
In the following “part 2” will present a custom pong service written in Go that will self-register to etcd and update available/healthy backends.
One thought on “Dynamic service discovery on top of Kubernetes – part 1 (etcd cluster)”