1
2
3
4
5
6
7
8
9
10
11
12
13
# Force reset Kubernetes — removes all pods, configs, and local data
sudo kubeadm reset -f

# Remove CNI configuration
sudo rm -rf /etc/cni/net.d

# Remove kubeconfig from root and your user
sudo rm -rf ~/.kube
rm -rf $HOME/.kube

# Remove Kubernetes package sources
sudo rm -f /etc/apt/sources.list.d/kubernetes.list    # Debian/Ubuntu
sudo rm -f /etc/yum.repos.d/kubernetes.repo            # CentOS/RHEL

1
2
3
4
5
6
# Unhold packages so they can be removed
sudo apt-mark unhold kubeadm kubelet kubectl

# Remove Kubernetes binaries
sudo apt remove --purge -y kubeadm kubelet kubectl
sudo apt autoremove -y

1
2
3
4
5
6
7
8
# Remove Docker packages
sudo apt remove --purge -y docker-ce docker-ce-cli containerd.io       # Debian/Ubuntu
sudo yum remove -y docker-ce docker-ce-cli containerd.io               # CentOS/RHEL
sudo rm -rf /var/lib/docker /var/lib/containerd

# Remove Docker configuration
sudo rm -rf /etc/docker
sudo rm -rf /etc/containerd

1
2
3
4
sudo rm -rf /etc/kubernetes
sudo rm -rf /var/lib/kubelet
sudo rm -rf /var/lib/etcd
sudo rm -rf /etc/cni

1
sudo rm -f /usr/local/bin/kubeadm /usr/local/bin/kubelet /usr/local/bin/kubectl

1
2
3
4
sudo iptables -F
sudo iptables -X
sudo iptables -t nat -F
sudo iptables -t mangle -F

1
2
3
4
5
# Remove the kubernetes sysctl config file
sudo rm -f /etc/sysctl.d/kubernetes.conf

# Apply the change
sudo sysctl --system

1
sudo reboot

1
2
3
4
# Replace with the actual hostname for this node
sudo hostnamectl set-hostname k8s-master-01   # On master 1
sudo hostnamectl set-hostname k8s-master-02   # On master 2
sudo hostnamectl set-hostname k8s-worker-01   # On worker 1, etc.

1
hostnamectl status

1
sudo vi /etc/hosts

1
2
# Ensure 127.0.1.1 points to the current node's hostname (NOT localhost)
echo "127.0.1.1 $(hostname)" | sudo tee -a /etc/hosts

1
2
3
sudo apt update
sudo apt upgrade -y
sudo apt autoremove -y

1
sudo yum update -y

1
2
3
4
5
6
7
8
9
# Temporarily set
sudo setenforce 0

# Permanently set in config
sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config
sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/sysconfig/selinux

# Verify
getenforce   # Should show "Permissive"

1
2
3
4
5
6
7
8
# Disable swap immediately
sudo swapoff -a

# Remove or comment out swap entries in /etc/fstab so it stays off after reboot
sudo sed -i '/ swap / s/^\(.*\)$/#\1/' /etc/fstab

# Verify
free -m        # Swap should show 0

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# Load modules immediately
sudo modprobe overlay
sudo modprobe br_netfilter
sudo modprobe bridge

# Create a config file so they load on boot
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
bridge
EOF

# If using Ceph/Rook, also load these:
sudo modprobe rbd
sudo modprobe ceph

cat <<EOF | sudo tee /etc/modules-load.d/rbd.conf
rbd
EOF

cat <<EOF | sudo tee /etc/modules-load.d/ceph.conf
ceph
EOF

# Verify modules are loaded
lsmod | grep -E "br_netfilter|overlay|bridge"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
cat <<EOF | sudo tee /etc/sysctl.d/kubernetes.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

# Also set in /etc/sysctl.conf for persistence
sudo sed -i '/^net.bridge.bridge-nf-call-iptables=/d' /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-iptables=1" | sudo tee -a /etc/sysctl.conf
sudo sed -i '/^net.bridge.bridge-nf-call-ip6tables=/d' /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-ip6tables=1" | sudo tee -a /etc/sysctl.conf

# Apply sysctl settings
sudo sysctl --system

# Verify
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

1
2
3
4
5
6
7
cat <<EOF | sudo tee /etc/sysctl.d/99-inotify-limits.conf
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 1024
fs.inotify.max_queued_events = 16384
EOF

sudo sysctl --system

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# Increase system-wide file descriptors
sudo sed -i '/^fs.file-max=/d' /etc/sysctl.conf
echo "fs.file-max=999999" | sudo tee -a /etc/sysctl.conf
sudo sysctl --system

# Add security limits
cat <<EOF | sudo tee -a /etc/security/limits.conf
*    hard    nofile    999999
*    soft    nofile    256000
EOF

# Enable PAM limits (Debian/Ubuntu)
grep -q "pam_limits.so" /etc/pam.d/common-session || echo "session required pam_limits.so" | sudo tee -a /etc/pam.d/common-session
grep -q "pam_limits.so" /etc/pam.d/common-session-noninteractive || echo "session required pam_limits.so" | sudo tee -a /etc/pam.d/common-session-noninteractive

# Enable PAM limits (CentOS/RHEL)
grep -q "pam_limits.so" /etc/pam.d/system-auth || echo "session required pam_limits.so" | sudo tee -a /etc/pam.d/system-auth

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# Global systemd limits
sudo sed -i 's/^#DefaultLimitNOFILE=/DefaultLimitNOFILE=999999/' /etc/systemd/system.conf
sudo sed -i 's/^#DefaultLimitNOFILE=/DefaultLimitNOFILE=999999/' /etc/systemd/user.conf

# Docker service override
sudo mkdir -p /etc/systemd/system/docker.service.d
cat <<EOF | sudo tee /etc/systemd/system/docker.service.d/override.conf
[Service]
LimitNOFILE=999999
EOF

# Kubelet service override
sudo mkdir -p /etc/systemd/system/kubelet.service.d
cat <<EOF | sudo tee /etc/systemd/system/kubelet.service.d/override.conf
[Service]
LimitNOFILE=999999
EOF

# Containerd service override
sudo mkdir -p /etc/systemd/system/containerd.service.d
cat <<EOF | sudo tee /etc/systemd/system/containerd.service.d/override.conf
[Service]
LimitNOFILE=999999
EOF

# Reload systemd
sudo systemctl daemon-reexec

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# CentOS/RHEL — disable firewalld
sudo systemctl stop firewalld
sudo systemctl disable firewalld

# Debian/Ubuntu — disable ufw
sudo systemctl stop ufw
sudo systemctl disable ufw
sudo ufw disable

# Flush all iptables rules
sudo iptables -F
sudo iptables -t nat -F
sudo iptables -t mangle -F
sudo ip6tables -F

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Install prerequisites
sudo apt install -y curl ca-certificates

# Create keyrings directory
sudo install -m 0755 -d /etc/apt/keyrings

# Add Docker's GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg

# Add Docker repository
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo \"$VERSION_CODENAME\") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

# Update and install
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

1
2
3
4
5
6
7
8
# Install dependencies
sudo yum install -y dnf-plugins-core

# Add Docker repository
sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

# Install Docker
sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

1
2
3
4
5
6
7
8
9
10
11
12
13
sudo mkdir -p /etc/docker
cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "bip": "172.30.0.1/24",
  "iptables": false,
  "ip-masq": false,
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
  }
}
EOF

1
2
3
4
sudo systemctl enable containerd
sudo systemctl start containerd
sudo systemctl enable docker
sudo systemctl start docker

1
2
3
4
5
6
7
8
9
# Generate default containerd config
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

# Enable SystemdCgroup
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

# Restart containerd
sudo systemctl restart containerd

1
2
sudo mkdir -p /run/systemd/resolve/
sudo ln -sf /run/NetworkManager/resolv.conf /run/systemd/resolve/resolv.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# Choose your Kubernetes version
K8S_VERSION="1.31"

# Create keyrings directory
sudo mkdir -p /etc/apt/keyrings

# Add Kubernetes GPG key
curl -fsSL https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
sudo chmod a+r /etc/apt/keyrings/kubernetes-apt-keyring.gpg

# Add Kubernetes repository
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list

# Update and install
sudo apt update
sudo apt install -y kubelet kubeadm kubectl

# Hold packages to prevent accidental upgrades
sudo apt-mark hold kubelet kubeadm kubectl

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
K8S_VERSION="1.31"

# Add Kubernetes repository
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

# Install
sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes

1
2
sudo systemctl enable kubelet
sudo systemctl start kubelet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Check Docker
docker --version

# Check Kubernetes components
kubelet --version
kubeadm version
kubectl version --client

# Check file descriptor limits
ulimit -n
grep "nofile" /etc/security/limits.conf

# Check modules
lsmod | grep -E "br_netfilter|overlay"

1
2
3
4
5
sudo apt install -y apparmor apparmor-utils

# Enable AppArmor in GRUB
sudo sed -i 's/GRUB_CMDLINE_LINUX="\(.*\)"/GRUB_CMDLINE_LINUX="\1 apparmor=1 security=apparmor"/' /etc/default/grub
sudo update-grub

1
2
3
4
5
6
# Prevent core dumps via limits.conf
echo "* hard core 0" | sudo tee -a /etc/security/limits.conf

# Disable suid core dumps
echo "fs.suid_dumpable = 0" | sudo tee -a /etc/sysctl.d/99-cis.conf
sudo sysctl -w fs.suid_dumpable=0

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Create admin groups
sudo groupadd --force sudo
sudo groupadd --force kubernetes-admin

# Restrict SSH to these groups
echo "AllowGroups sudo kubernetes-admin" | sudo tee -a /etc/ssh/sshd_config

# Configure strong SSH MAC algorithms
echo "MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512,hmac-sha2-256" | sudo tee -a /etc/ssh/sshd_config

# Disable root login
sudo sed -i 's/^#PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config
sudo sed -i 's/^PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config

# Also check and update included configs
for f in /etc/ssh/sshd_config.d/*.conf; do
  [ -f "$f" ] && sudo sed -i 's/^PermitRootLogin.*/PermitRootLogin no/' "$f"
done

# Restart SSH
sudo systemctl restart sshd

1
echo "Authorized users only. All activity may be monitored and reported." | sudo tee /etc/issue.net

1
echo "Defaults logfile=/var/log/sudo.log" | sudo EDITOR='tee -a' visudo

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# Install pwquality
sudo apt install -y libpam-pwquality

# Configure password quality rules
cat <<EOF | sudo tee -a /etc/security/pwquality.conf
difok = 3
minlen = 14
dcredit = -1
ucredit = -1
lcredit = -1
ocredit = -1
maxrepeat = 3
maxsequence = 3
dictcheck = 1
EOF

# Ensure PAM enforces password quality
grep -q "pam_pwquality.so" /etc/pam.d/common-password || echo "password requisite pam_pwquality.so retry=3 enforce_for_root" | sudo tee -a /etc/pam.d/common-password

1
sudo sed -i 's/\(pam_unix\.so.*\)nullok\(.*\)/\1\2/' /etc/pam.d/common-password

1
echo 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' | sudo tee /etc/environment

1
2
3
4
5
6
cat <<EOF | sudo tee -a /etc/profile

# Set session timeout (15 minutes)
readonly TMOUT=900
export TMOUT
EOF

1
2
3
4
5
6
7
8
9
10
sudo apt install -y aide aide-common

# Initialize AIDE database (this may take a while)
sudo aideinit

# Move the new database into place
sudo mv /var/lib/aide/aide.db.new /var/lib/aide/aide.db

# Configure daily AIDE checks
echo "0 5 * * * /usr/bin/aide.wrapper --check" | sudo crontab -

1
2
3
4
5
6
7
8
9
10
11
12
13
14
sudo apt install -y systemd-journal-remote

# Configure journal upload
cat <<EOF | sudo tee /etc/systemd/journal-upload.conf
[Upload]
URL=http://<YOUR-LOG-SERVER-IP>:19532
ServerKeyFile=/etc/ssl/private/journal-upload.pem
ServerCertificateFile=/etc/ssl/certs/journal-upload.pem
TrustedCertificateFile=/etc/ssl/ca/trusted.pem
EOF

# Enable and start the upload service
sudo systemctl enable systemd-journal-upload.service
sudo systemctl start systemd-journal-upload.service

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Restrict permissions on kubelet service files
sudo chmod 0600 /etc/systemd/system/kubelet.service
sudo chmod 0600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

# Restrict kubelet config
sudo chmod 0600 /var/lib/kubelet/config.yaml

# Restrict proxy kubeconfig
sudo chmod 0600 /etc/kubernetes/proxy.conf

# Restrict CA certificate
sudo chmod 0600 /etc/kubernetes/pki/ca.crt

# Restrict kubelet TLS certs
sudo chmod 0600 /var/lib/kubelet/pki/kubelet.crt
sudo chmod 0600 /var/lib/kubelet/pki/kubelet.key

1
2
3
4
5
6
7
8
9
10
# Remove any existing containerd config and regenerate
sudo rm -f /etc/containerd/config.toml
sudo containerd config default | sudo tee /etc/containerd/config.toml

# Enable SystemdCgroup
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

# Restart containerd and kubelet
sudo systemctl restart containerd
sudo systemctl restart kubelet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Create config directory
sudo mkdir -p /etc/kubernetes

# Create kubeadm config
cat <<EOF | sudo tee /etc/kubernetes/kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
  podSubnet: 10.244.0.0/16
controlPlaneEndpoint: "k8s-api.example.com:6443"
EOF

1
2
3
4
5
6
7
# Initialize certificates CA phase first
sudo kubeadm init phase certs ca

# Then initialize the cluster
sudo kubeadm init \
  --config=/etc/kubernetes/kubeadm-config.yaml \
  --upload-certs 2>&1 | tee cluster_initialized.log

1
2
3
4
5
6
7
8
9
# Create kube directory
mkdir -p $HOME/.kube

# Copy admin config
sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# Verify access
kubectl get nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# Set architecture
ARCH=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')
CILIUM_VERSION="v0.18.2"

# Download Cilium CLI
curl -L "https://github.com/cilium/cilium-cli/releases/download/${CILIUM_VERSION}/cilium-linux-${ARCH}.tar.gz" -o /tmp/cilium.tar.gz
curl -L "https://github.com/cilium/cilium-cli/releases/download/${CILIUM_VERSION}/cilium-linux-${ARCH}.tar.gz.sha256sum" -o /tmp/cilium.sha256sum

# Verify checksum
cd /tmp && sha256sum -c cilium.sha256sum

# Extract
sudo tar xzvf /tmp/cilium.tar.gz -C /usr/local/bin

# Clean up
rm -f /tmp/cilium.tar.gz /tmp/cilium.sha256sum

# Install Cilium in the cluster (may take 2-3 minutes)
cilium install --version 1.16.4

# Verify Cilium is running
cilium status --wait

1
2
3
4
5
6
# Wait a moment for everything to settle
sleep 30

# Check cluster status
kubectl get nodes
kubectl get pods -A

1
2
3
4
5
6
7
8
9
10
# Generate worker join command (valid for 2 hours)
echo "--- Worker Join Command ---"
sudo kubeadm token create --print-join-command

# Generate master join command (includes certificate key)
echo "--- Master Join Command ---"
CERT_KEY=$(sudo kubeadm init phase upload-certs --upload-certs | grep -A 1 'certificate key' | tail -n 1)
sudo kubeadm token create --print-join-command --certificate-key $CERT_KEY
echo ""
echo "Add --control-plane flag to the master join command above"

1
2
3
4
5
sudo kubeadm join k8s-api.example.com:6443 \
  --token <your-token> \
  --discovery-token-ca-cert-hash sha256:<hash> \
  --control-plane \
  --certificate-key <cert-key>

1
2
3
mkdir -p $HOME/.kube
sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

1
kubectl get nodes

1
2
3
sudo kubeadm join k8s-api.example.com:6443 \
  --token <your-token> \
  --discovery-token-ca-cert-hash sha256:<hash>

1
kubectl get nodes -o wide

1
2
3
4
5
# Debian/Ubuntu
sudo apt install -y keepalived haproxy

# CentOS/RHEL
sudo yum install -y keepalived haproxy

1
2
3
4
5
6
7
8
9
10
11
12
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state MASTER
    interface enp1s0                  # Replace with your network interface
    virtual_router_id 51
    priority 100                      # Highest priority
    advert_int 1
    virtual_ipaddress {
        192.168.1.100                 # Replace with your chosen VIP
    }
}
EOF

1
2
3
4
5
6
7
8
9
10
11
12
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state BACKUP
    interface enp1s0                  # Replace with your network interface
    virtual_router_id 51
    priority 90                       # Lower than master 1
    advert_int 1
    virtual_ipaddress {
        192.168.1.100                 # Replace with your chosen VIP
    }
}
EOF

1
2
3
4
5
6
7
8
9
10
11
12
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state BACKUP
    interface enp1s0                  # Replace with your network interface
    virtual_router_id 51
    priority 80                       # Lower than master 2
    advert_int 1
    virtual_ipaddress {
        192.168.1.100                 # Replace with your chosen VIP
    }
}
EOF

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# Get the list of master IPs
# In this example: 192.168.1.10, 192.168.1.11, 192.168.1.12

cat <<EOF | sudo tee /etc/haproxy/haproxy.cfg
global
    log /dev/log local0
    maxconn 2000
    user haproxy
    group haproxy

defaults
    log     global
    mode    tcp
    timeout connect 10s
    timeout client  1m
    timeout server  1m

frontend kube-apiserver
    bind 192.168.1.100:6443           # VIP:port
    default_backend kube-apiserver-backend

backend kube-apiserver-backend
    mode tcp
    server master-1 192.168.1.10:6443 check
    server master-2 192.168.1.11:6443 check
    server master-3 192.168.1.12:6443 check
EOF

1
2
3
4
5
6
7
8
sudo systemctl enable keepalived
sudo systemctl start keepalived
sudo systemctl enable haproxy
sudo systemctl start haproxy

# Verify they are running
sudo systemctl status keepalived
sudo systemctl status haproxy

1
2
3
4
# Check which master currently owns the VIP
ip addr show | grep 192.168.1.100

# You should see the VIP on the MASTER node

1
2
3
4
5
6
7
8
9
10
11
# Get the current ClusterConfiguration
kubectl get cm kubeadm-config -n kube-system -o jsonpath='{.data.ClusterConfiguration}' > /tmp/current-config.yaml

# Update controlPlaneEndpoint to use the VIP hostname
sed -i "s/controlPlaneEndpoint: .*/controlPlaneEndpoint: k8s-api.example.com:6443/" /tmp/current-config.yaml

# Patch the ConfigMap
kubectl patch configmap kubeadm-config -n kube-system --patch "{\"data\":{\"ClusterConfiguration\":\"$(cat /tmp/current-config.yaml | sed ':a;N;$!ba;s/\n/\\n/g')\"}}"

# Clean up
rm /tmp/current-config.yaml

1
2
3
4
5
# Update all kubeconfig files to use the VIP hostname
sudo sed -i 's|server: https://.*:6443|server: https://k8s-api.example.com:6443|' /etc/kubernetes/admin.conf
sudo sed -i 's|server: https://.*:6443|server: https://k8s-api.example.com:6443|' /etc/kubernetes/controller-manager.conf
sudo sed -i 's|server: https://.*:6443|server: https://k8s-api.example.com:6443|' /etc/kubernetes/scheduler.conf
sudo sed -i 's|server: https://.*:6443|server: https://k8s-api.example.com:6443|' $HOME/.kube/config

1
2
3
# Update kubelet configs to use the VIP hostname
sudo sed -i 's|server: https://.*:6443|server: https://k8s-api.example.com:6443|' /etc/kubernetes/kubelet.conf
sudo sed -i 's|server: https://.*:6443|server: https://k8s-api.example.com:6443|' /var/lib/kubelet/kubeconfig

1
sudo systemctl restart kubelet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# Backup existing certificates
sudo cp /etc/kubernetes/pki/apiserver.crt /etc/kubernetes/pki/apiserver.crt.bak-$(date +%Y%m%d%H%M%S)
sudo cp /etc/kubernetes/pki/apiserver.key /etc/kubernetes/pki/apiserver.key.bak-$(date +%Y%m%d%H%M%S)

# Remove old certificates
sudo rm -f /etc/kubernetes/pki/apiserver.crt /etc/kubernetes/pki/apiserver.key

# Create certificate config with SANs
cat <<EOF | sudo tee /root/kubeadm-cert-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: "k8s-api.example.com:6443"
apiServer:
  certSANs:
    - "192.168.1.100"            # VIP
    - "k8s-api.example.com"       # VIP hostname
    - "10.96.0.1"                 # Kubernetes service IP
    - "kubernetes"
    - "kubernetes.default"
    - "kubernetes.default.svc"
    - "kubernetes.default.svc.cluster.local"
    - "localhost"
    - "127.0.0.1"
    - "192.168.1.10"              # Master 1 IP
    - "192.168.1.11"              # Master 2 IP
    - "192.168.1.12"              # Master 3 IP
EOF

# Regenerate certificates
sudo kubeadm init phase certs apiserver --config /root/kubeadm-cert-config.yaml

# Verify the new certificate includes the VIP
openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text | grep -A1 "Subject Alternative Name"

1
2
3
4
5
6
7
8
9
10
11
12
# Restart kubelet
sudo systemctl restart kubelet

# Wait for API server to be ready
sleep 10

# Test connectivity via the VIP
kubectl get nodes --server=https://k8s-api.example.com:6443

# If that works, test VIP failover by temporarily stopping keepalived on the master
# sudo systemctl stop keepalived
# Then check that the VIP moves to another master

1
2
# Check expiry of all certificates
sudo kubeadm certs check-expiration

1
2
3
4
5
# Renew all certificates
sudo kubeadm certs renew all

# Restart control plane components
sudo systemctl restart kubelet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# On all masters:
# 1. Backup old certificates
sudo cp -r /etc/kubernetes/pki /etc/kubernetes/pki.bak-$(date +%Y%m%d%H%M%S)

# 2. Remove the apiserver cert and key
sudo rm -f /etc/kubernetes/pki/apiserver.crt /etc/kubernetes/pki/apiserver.key

# 3. Regenerate with your config
sudo kubeadm init phase certs apiserver --config /root/kubeadm-cert-config.yaml

# 4. Restart kubelet
sudo systemctl restart kubelet

# 5. On the first master only — update the admin kubeconfig
sudo kubeadm init phase kubeconfig all --control-plane-endpoint k8s-api.example.com:6443

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# Stop kubelet
sudo systemctl stop kubelet

# Kill any processes on port 10250 (kubelet port)
sudo lsof -t -i:10250 | xargs -r sudo kill -9

# Force reset
sudo kubeadm reset --force

# Remove CNI and Kubernetes configs
sudo rm -rf /etc/cni/net.d
sudo rm -f /etc/kubernetes/kubelet.conf
sudo rm -f /etc/kubernetes/pki/ca.crt
sudo rm -f /etc/kubernetes/bootstrap-kubelet.conf
sudo rm -rf /etc/kubernetes/pki
sudo rm -rf /var/lib/kubelet/pki
sudo rm -f /var/lib/kubelet/config.yaml

# Reset iptables
sudo iptables -F
sudo iptables -t nat -F
sudo iptables -t mangle -F
sudo iptables -X

# Restart services
sudo systemctl restart containerd
sudo systemctl restart kubelet

1
2
3
4
5
# Get the name of the worker node
kubectl get nodes

# Delete the worker node (replace with actual name)
kubectl delete node k8s-worker-01

1
2
# Reboot all nodes (masters and workers)
sudo reboot

1
2
3
4
# On any master:
kubectl get nodes
kubectl get pods -A
cilium status

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# Cluster status
kubectl cluster-info
kubectl get nodes -o wide
kubectl get pods -A
kubectl get svc -A

# Cilium status
cilium status
cilium connectivity test

# Certificate expiry
sudo kubeadm certs check-expiration

# HAProxy status
sudo systemctl status haproxy
sudo haproxy -f /etc/haproxy/haproxy.cfg -c   # Check config

# Keepalived status
sudo systemctl status keepalived
ip addr show | grep <VIP>    # Check which node has the VIP

# System checks
free -m                       # Memory
df -h                         # Disk
uptime                        # How long running
sudo sysctl net.ipv4.ip_forward  # Verify IP forwarding

1
2
3
4
5
6
7
8
9
10
11
12
13
    # Add Percona repository
    sudo percona-release setup ppg12

    # Install required packages
    sudo apt update
    sudo apt install -y \
        percona-postgresql-12 \
        percona-patroni \
        etcd \
        percona-haproxy \
        keepalived \
        python3-pip \
        python3-dev

1
2
3
4
5
6
7
8
9
10
11
    # Set node name and IP
    export NODE_NAME=$(hostname -f)
    export NODE_IP=$(hostname -i | awk '{print $1}')

    # Set PostgreSQL paths
    export DATA_DIR="/var/lib/postgresql/12/main"
    export PG_BIN_DIR="/usr/lib/postgresql/12/bin"

    # Set cluster information
    export NAMESPACE="kyc"
    export SCOPE="kyc"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    echo "
    namespace: ${NAMESPACE}
    scope: ${SCOPE}
    name: ${NODE_NAME}

    restapi:
        listen: 0.0.0.0:8008
        connect_address: ${NODE_IP}:8008

    etcd3:
        host: ${NODE_IP}:2379

    bootstrap:
    # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
    dcs:
        ttl: 30
        loop_wait: 10
        retry_timeout: 10
        maximum_lag_on_failover: 1048576

        postgresql:
            use_pg_rewind: true
            use_slots: true
            parameters:
                wal_level: replica
                hot_standby: "on"
                wal_keep_segments: 10
                max_wal_senders: 5
                max_replication_slots: 10
                wal_log_hints: "on"
                logging_collector: 'on'
                max_wal_size: '10GB'
                archive_mode: "on"
                archive_timeout: 600s
                archive_command: "cp -f %p /home/postgres/archived/%f"

    # some desired options for 'initdb'
    initdb: # Note: It needs to be a list (some options need values, others are switches)
        - encoding: UTF8
        - data-checksums

    pg_hba: # Add following lines to pg_hba.conf after running 'initdb'
        - host replication replicator 127.0.0.1/32 trust
        - host replication replicator 0.0.0.0/0 md5
        - host all all 0.0.0.0/0 md5
        - host all all ::0/0 md5

    # Some additional users which needs to be created after initializing new cluster
    users:
        admin:
            password: <REDACTED_PASSWORD>
            options:
                - createrole
                - createdb
        percona:
            password: <REDACTED_PASSWORD>
            options:
                - createrole
                - createdb 

    postgresql:
        cluster_name: kyc
        listen: 0.0.0.0:5432
        connect_address: ${NODE_IP}:5432
        data_dir: ${DATA_DIR}
        bin_dir: ${PG_BIN_DIR}
        pgpass: /tmp/pgpass0
        authentication:
            replication:
                username: replicator
                password: <REDACTED_PASSWORD>
            superuser:
                username: postgres
                password: <REDACTED_PASSWORD>
        parameters:
            unix_socket_directories: "/var/run/postgresql/"
        create_replica_methods:
            - basebackup
        basebackup:
            checkpoint: 'fast'

    tags:
        nofailover: false
        noloadbalance: false
        clonefrom: false
        nosync: false
    " | sudo tee -a /etc/patroni/patroni.yml

1
2
3
    sudo systemctl daemon-reload
    sudo systemctl enable patroni
    sudo systemctl start patroni

1
    sudo journalctl -fu patroni

1
    sudo apt install percona-haproxy

1
2
    sudo systemctl restart haproxy
    sudo systemctl enable haproxy

1
    sudo journalctl -u haproxy.service -n 100 -f

1
    psql -h <REDACTED_IP> -p 5433 -U postgres -d postgres

1
2
3
4
5
6
7
8
9
10
    # Log into the previous PostgreSQL cluster and switch to postgres bash user.
    sudo su - postgres

    # Backup the <REDACTED_USERNAME> schema data
    pg_dump -h localhost -U kyc -d postgres -f <REDACTED_USERNAME>_full.sql

    # Backup the schema only data of kyc, <REDACTED_USERNAME> and <REDACTED_USERNAME>
    pg_dump -h localhost -U kyc -d postgres -s -f kyc_schema.sql
    pg_dump -h localhost -U kyc -d postgres -s -f <REDACTED_USERNAME>_schema.sql
    pg_dump -h localhost -U kyc -d postgres -s -f <REDACTED_USERNAME>_schema.sql

1
2
    # Backup the kyc.asp_config and kyc.asp_config_id_seq tables
    pg_dump -h localhost -U kyc -d postgres -t kyc.asp_config -t kyc.asp_config_id_seq -f kyc_tables.sql

1
2
3
4
5
# Add to /etc/hosts on <REDACTED_HOSTNAME>
<REDACTED_IP> <REDACTED_HOSTNAME>
<REDACTED_IP> <REDACTED_HOSTNAME>
<REDACTED_IP> <REDACTED_HOSTNAME>
<REDACTED_IP> <REDACTED_HOSTNAME>

1
2
sudo apt update
sudo apt install -y percona-pgbackrest

1
2
3
sudo mkdir -p /var/lib/postgresql/pgbackup
sudo chown postgres:postgres /var/lib/postgresql/pgbackup
sudo chmod 750 /var/lib/postgresql/pgbackup

1
2
3
4
5
6
7
8
9
10
11
# Create SSH directory
sudo mkdir -p /var/lib/postgresql/.ssh
sudo chown postgres:postgres /var/lib/postgresql/.ssh
sudo chmod 700 /var/lib/postgresql/.ssh

# Generate SSH key pair (as postgres user)
sudo -u postgres ssh-keygen -t rsa -b 4096 -f /var/lib/postgresql/.ssh/id_rsa -N ""

# Set proper permissions
sudo chmod 600 /var/lib/postgresql/.ssh/id_rsa
sudo chmod 600 /var/lib/postgresql/.ssh/id_rsa.pub

1
2
# Display public key (save this for configuring PostgreSQL nodes)
sudo cat /var/lib/postgresql/.ssh/id_rsa.pub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# Create configuration file
sudo bash -c "cat > /etc/pgbackrest.conf << 'EOL'
[global]
# Repository configuration
repo1-path=/var/lib/postgresql/pgbackup
repo1-retention-archive-type=full
repo1-retention-full=1

# Server options
process-max=12
log-level-console=info
log-level-file=info
start-fast=y
delta=y
backup-standby=y

[kyc]
pg1-host=<REDACTED_HOSTNAME>
pg1-host-user=postgres
pg1-port=5432
pg1-path=/var/lib/postgresql/12/main
pg1-socket-path=/var/run/postgresql

pg2-host=<REDACTED_HOSTNAME>
pg2-host-user=postgres
pg2-port=5432
pg2-path=/var/lib/postgresql/12/main
pg2-socket-path=/var/run/postgresql

pg3-host=<REDACTED_HOSTNAME>
pg3-host-user=postgres
pg3-port=5432
pg3-path=/var/lib/postgresql/12/main
pg3-socket-path=/var/run/postgresql
EOL"

sudo chown postgres:postgres /etc/pgbackrest.conf
sudo chmod 640 /etc/pgbackrest.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
sudo bash -c "cat > /etc/systemd/system/pgbackrest.service << 'EOL'
[Unit]
Description=pgBackRest Server
After=network.target

[Service]
Type=simple
User=postgres
Restart=always
RestartSec=1
ExecStart=/usr/bin/pgbackrest server
ExecReload=/bin/kill -HUP \$MAINPID

[Install]
WantedBy=multi-user.target
EOL"

1
2
3
sudo systemctl daemon-reload
sudo systemctl enable pgbackrest
sudo systemctl start pgbackrest

1
2
# Add to /etc/hosts on each PostgreSQL node
<REDACTED_IP> <REDACTED_HOSTNAME>

1
2
sudo apt update
sudo apt install -y percona-pgbackrest

1
2
3
4
5
6
7
8
9
# Create SSH directory
sudo mkdir -p /var/lib/postgresql/.ssh
sudo chown postgres:postgres /var/lib/postgresql/.ssh
sudo chmod 700 /var/lib/postgresql/.ssh

# Create authorized_keys file
sudo touch /var/lib/postgresql/.ssh/authorized_keys
sudo chown postgres:postgres /var/lib/postgresql/.ssh/authorized_keys
sudo chmod 600 /var/lib/postgresql/.ssh/authorized_keys

1
2
# Add the pgBackRest server's public key
sudo bash -c "echo 'PASTE_PGBACKREST_PUBLIC_KEY_HERE' >> /var/lib/postgresql/.ssh/authorized_keys"

1
2
# Generate SSH key pair (as postgres user)
sudo -u postgres ssh-keygen -t rsa -b 4096 -f /var/lib/postgresql/.ssh/id_rsa -N ""

1
2
# Display public key (to add to pgBackRest server)
sudo cat /var/lib/postgresql/.ssh/id_rsa.pub

1
2
# On <REDACTED_HOSTNAME>
sudo bash -c "echo 'PASTE_POSTGRES_NODE_PUBLIC_KEY_HERE' >> /var/lib/postgresql/.ssh/authorized_keys"

1
2
3
# Add pgBackRest host to known_hosts
sudo -u postgres ssh-keyscan -t rsa <REDACTED_HOSTNAME> >> /var/lib/postgresql/.ssh/known_hosts
sudo chmod 600 /var/lib/postgresql/.ssh/known_hosts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
sudo bash -c "cat > /etc/pgbackrest.conf << 'EOL'
[global]
repo1-host=<REDACTED_HOSTNAME>
repo1-host-user=postgres

# General options
process-max=16
log-level-console=info
log-level-file=debug

[kyc]
pg1-path=/var/lib/postgresql/12/main
EOL"

sudo chown postgres:postgres /etc/pgbackrest.conf
sudo chmod 640 /etc/pgbackrest.conf

1
2
3
4
patronictl -c /etc/patroni/patroni.yml edit-config -s postgresql.parameters.archive_mode=on \
-s postgresql.parameters.archive_command="pgbackrest --stanza=kyc archive-push %p" \
-s postgresql.recovery_conf.restore_command="pgbackrest --config=/etc/pgbackrest.conf --stanza=kyc archive-get %f %p" \
--force

1
patronictl -c /etc/patroni/patroni.yml reload kyc --force

1
sudo -iu postgres pgbackrest --stanza=kyc stanza-create

1
sudo -iu postgres pgbackrest --stanza=kyc --type=full backup

1
sudo -iu postgres pgbackrest --stanza=kyc info

1
sudo systemctl status pgbackrest

1
sudo -u postgres pgbackrest --stanza=kyc check

1
2
sudo -u postgres pgbackrest --stanza=kyc --type=full backup
sudo -u postgres pgbackrest --stanza=kyc info

1
2
3
4
# Edit the crontab for the postgres user
sudo crontab -u postgres -e
# Add the following line to schedule a daily backup at 2 AM
0 2 * * * /usr/bin/pgbackrest --stanza=kyc --type=full backup

1
2
# Check the backup logs
sudo tail -f /var/log/pgbackrest/pgbackrest.log

1
2
3
4
5
6
7
8
# Stop the PostgreSQL service on the primary node
sudo systemctl stop patroni
# Restore the backup
sudo -u postgres pgbackrest --stanza=kyc restore
# Start the PostgreSQL service
sudo systemctl start patroni
# Check the status of the Patroni cluster
patronictl -c /etc/patroni/patroni.yml list

1
2
3
4
5
# Update package lists
sudo apt update

# Install prerequisites
sudo apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg

1
2
3
4
5
# Add Docker's official GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

# Add Docker repository
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

1
2
3
4
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io
sudo systemctl enable docker
sudo systemctl start docker

1
curl -fsSL https://raw.githubusercontent.com/percona/<REDACTED_USERNAME>/refs/heads/v3/get-<REDACTED_USERNAME>.sh | sudo bash

1
2
# You can monitor the container status
sudo docker ps | grep <REDACTED_USERNAME>-server

1
sudo docker exec -t <REDACTED_USERNAME>-server change-admin-password <REDACTED_PASSWORD>

1
2
wget -O - https://repo.percona.com/apt/percona-release_latest.generic_all.deb > /tmp/percona-release.deb
sudo dpkg -i /tmp/percona-release.deb

1
sudo percona-release enable <REDACTED_USERNAME>3-client release

1
2
sudo apt update
sudo apt install -y <REDACTED_USERNAME>-client

1
sudo apt install -y percona-pg-stat-monitor12

1
sudo -u postgres psql -c "CREATE USER <REDACTED_USERNAME> WITH SUPERUSER PASSWORD '<REDACTED_PASSWORD>';" || sudo -u postgres psql -c "ALTER USER <REDACTED_USERNAME> WITH SUPERUSER PASSWORD '<REDACTED_PASSWORD>';"

1
2
# Add this line to /var/lib/postgresql/12/main/pg_hba.conf
echo "local   all             <REDACTED_USERNAME>                                     md5" | sudo tee -a /var/lib/postgresql/12/main/pg_hba.conf

1
2
3
4
# Update Patroni configuration
sudo patronictl -c /etc/patroni/patroni.yml edit-config -s postgresql.parameters.shared_preload_libraries="pg_stat_monitor" \
-s postgresql.parameters.pg_stat_monitor.pgsm_query_max_len=2048 \
--force

1
2
# Reload Patroni configuration
sudo patronictl -c /etc/patroni/patroni.yml reload kyc --force

1
sudo -u postgres psql -d postgres -c 'CREATE EXTENSION IF NOT EXISTS pg_stat_monitor;'

1
sudo <REDACTED_USERNAME>-admin config --server-insecure-tls --server-url=https://admin:<REDACTED_PASSWORD>@PMM_SERVER_IP:443 --force

1
2
3
4
5
# Remove any existing PostgreSQL monitoring (optional)
sudo <REDACTED_USERNAME>-admin remove postgresql || true

# Add PostgreSQL monitoring
sudo <REDACTED_USERNAME>-admin add postgresql --username=<REDACTED_USERNAME> --password=<REDACTED_PASSWORD>

1
sudo <REDACTED_USERNAME>-admin status

1
sudo <REDACTED_USERNAME>-admin list

1
2
3
4
5
6
7
8
9
10
sudo bash -c "cat > /usr/local/bin/<REDACTED_USERNAME>-maintenance.sh << 'EOL'
#!/bin/bash
# Purge old monitoring data (adjust retention as needed)
<REDACTED_USERNAME>-admin maintenance --retention 14d

# Check PMM client status
<REDACTED_USERNAME>-admin list
EOL"

sudo chmod +x /usr/local/bin/<REDACTED_USERNAME>-maintenance.sh

1
(crontab -l 2>/dev/null; echo "0 2 * * 0 /usr/local/bin/<REDACTED_USERNAME>-maintenance.sh > /var/log/<REDACTED_USERNAME>-maintenance.log 2>&1") | crontab -

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Add PostgreSQL repository (EL-9)
sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm

# Disable built-in PostgreSQL module
sudo dnf -qy module disable postgresql

# Install PostgreSQL 17
sudo dnf install -y postgresql17-server postgresql17-contrib

# Prepare PGDATA directory
sudo mkdir -p /data/pgdata/17/data
sudo chown -R postgres:postgres /data/pgdata

# Set PGDATA in systemd
sudo systemctl edit postgresql-17.service

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
sudo systemctl daemon-reload

# Initialize database cluster
sudo /usr/pgsql-17/bin/postgresql-17-setup initdb

# Enable and start PostgreSQL
sudo systemctl enable postgresql-17.service
sudo systemctl start postgresql-17.service

# Add Pgpool repository (4.5 for PG17)
sudo yum install -y https://www.pgpool.net/yum/rpms/4.5/redhat/rhel-9-x86_64/pgpool-II-release-4.5-1.noarch.rpm

# If GPG check issue arises
sudo vim /etc/yum.repos.d/pgpool-II-release-45.repo
# Set: gpgcheck=0

# Install Pgpool for PostgreSQL 17
sudo yum install -y pgpool-II-pg17 pgpool-II-pg17-extensions pgpool-II-pg17-devel

# If dependency issues arise
wget https://rpmfind.net/linux/centos-stream/9-stream/CRB/x86_64/os/Packages/libmemcached-awesome-1.1.0-12.el9.x86_64.rpm
wget https://rpmfind.net/linux/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libcap-2.48-9.el9.x86_64.rpm
sudo yum install ./libmemcached-awesome-*.rpm ./libcap-*.rpm

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Create the PostgreSQL data directory
sudo mkdir -p /data/pgdata/17/data
sudo chown -R postgres:postgres /data/pgdata
sudo chmod 700 /data/pgdata/17/data

# Create log directories for Pgpool
sudo mkdir -p /var/log/pgpool_log/oiddir
sudo chown -R postgres:postgres /var/log/pgpool_log
sudo chmod 755 /var/log/pgpool_log

# Create Pgpool PID directory
sudo mkdir -p /var/run/pgpool-II
sudo chown postgres:postgres /var/run/pgpool-II
sudo chmod 755 /var/run/pgpool-II

1
sudo systemctl edit postgresql-17.service

1
sudo systemctl daemon-reload

1
2
3
sudo /usr/pgsql-17/bin/postgresql-17-setup initdb
sudo systemctl enable postgresql-17.service
sudo systemctl start postgresql-17.service

1
2
3
4
# Add include_dir to the bottom of postgresql.conf (one-time only)
echo "include_dir = 'conf.d'" | sudo tee -a /data/pgdata/17/data/postgresql.conf
sudo mkdir -p /data/pgdata/17/data/conf.d
sudo chown postgres:postgres /data/pgdata/17/data/conf.d

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
sudo -u postgres tee /data/pgdata/17/data/conf.d/01-performance.conf << 'EOF'
# =============================================================
# PostgreSQL 17 Tuning — 12 vCPU / 20 GB RAM / 80 GB Storage
# =============================================================

# ============================================================
# Connection Settings
# ============================================================
max_connections = 3200
reserved_connections = 10
superuser_reserved_connections = 5
password_encryption = scram-sha-256

# ============================================================
# Memory
# ============================================================
shared_buffers = 8GB
huge_pages = try
work_mem = 4MB
maintenance_work_mem = 2GB
autovacuum_work_mem = 2GB
temp_buffers = 16MB
effective_cache_size = 24GB

# ============================================================
# Parallelism
# ============================================================
max_worker_processes = 16
max_parallel_workers_per_gather = 4
max_parallel_workers = 16
max_parallel_maintenance_workers = 4
parallel_setup_cost = 500

# ============================================================
# Query Planner
# ============================================================
random_page_cost = 1.1
effective_io_concurrency = 200
cpu_tuple_cost = 0.01
cpu_index_tuple_cost = 0.005
default_statistics_target = 100
jit = off

# ============================================================
# WAL & Checkpoints
# ============================================================
wal_level = replica
wal_buffers = 16MB
wal_compression = lz4
wal_keep_size = 4096
wal_sender_timeout = 60s
min_wal_size = 1GB
max_wal_size = 4GB
checkpoint_timeout = 15min
checkpoint_completion_target = 0.9
synchronous_commit = local

# ============================================================
# Replication
# ============================================================
max_wal_senders = 10
max_replication_slots = 10
hot_standby = on
hot_standby_feedback = on

# ============================================================
# Autovacuum
# ============================================================
autovacuum = on
autovacuum_max_workers = 4
autovacuum_naptime = 30s
autovacuum_vacuum_cost_delay = 2ms
autovacuum_vacuum_scale_factor = 0.05
autovacuum_analyze_scale_factor = 0.02
log_autovacuum_min_duration = 250ms

# ============================================================
# Logging
# ============================================================
log_destination = 'stderr'
logging_collector = on
log_directory = 'log'
log_filename = 'postgresql-%a.log'
log_rotation_age = 1d
log_rotation_size = 1GB
log_truncate_on_rotation = on
log_min_duration_statement = 1000
log_checkpoints = on
log_connections = off
log_disconnections = off
log_lock_waits = on
log_temp_files = 10MB
log_error_verbosity = verbose
log_line_prefix = '%m [%p]: [%l-1] db=%d,user=%u,app=%a,client=%h'
log_statement = 'ddl'
log_replication_commands = on
log_timezone = 'Your/Timezone'

# ============================================================
# Session Behavior
# ============================================================
idle_in_transaction_session_timeout = 60000
lock_timeout = 0
statement_timeout = 0
track_io_timing = on
track_activity_query_size = 4096
EOF

1
2
3
4
5
6
7
8
9
10
sudo -u postgres tee /data/pgdata/17/data/conf.d/02-hba-rules.conf << 'EOF'
# This file does NOT override pg_hba.conf.
# HBA rules must be added directly to pg_hba.conf.
# This file documents what was added for reference only.
#
# Lines added to /data/pgdata/17/data/pg_hba.conf:
#   host replication repl_user 10.0.0.0/24 scram-sha-256
#   host all          pgpool   10.0.0.0/24 scram-sha-256
#   host all          app_user 10.0.0.0/24 scram-sha-256
EOF

1
2
3
4
5
6
7
sudo -u postgres tee -a /data/pgdata/17/data/pg_hba.conf << 'EOF'

# --- Custom rules ---
host replication repl_user 10.0.0.0/24 scram-sha-256
host all          pgpool   10.0.0.0/24 scram-sha-256
host all          app_user 10.0.0.0/24 scram-sha-256
EOF

1
sudo systemctl restart postgresql-17.service

1
2
sudo su - postgres
psql

1
exit

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# Stop PostgreSQL if running
sudo systemctl stop postgresql-17 || true

# Remove old data directory
sudo rm -rf /data/pgdata/17/data/*

# Take base backup from primary (includes conf.d, pg_hba.conf, everything)
sudo -u postgres pg_basebackup \
  -h 10.0.0.1 \
  -U repl_user \
  -p 5432 \
  -D /data/pgdata/17/data/ \
  -Fp -Xs -P -R
# Password: ChangeMe123!
# -R creates standby.signal and populates primary_conninfo automatically

# Start standby
sudo systemctl start postgresql-17
sudo systemctl enable postgresql-17

1
sudo -u postgres psql -c "SELECT client_addr, state, sync_state, sent_lsn, write_lsn, flush_lsn, replay_lsn FROM pg_stat_replication;"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# PostgreSQL
sudo firewall-cmd --permanent --add-port=5432/tcp

# Pgpool client port
sudo firewall-cmd --permanent --add-port=9999/tcp

# PCP admin port
sudo firewall-cmd --permanent --add-port=9898/tcp

# Watchdog
sudo firewall-cmd --permanent --add-port=9000/tcp

# Heartbeat (UDP)
sudo firewall-cmd --permanent --add-port=9694/udp

# Trust internal subnet
sudo firewall-cmd --permanent --zone=trusted --add-source=10.0.0.0/24

sudo firewall-cmd --reload
sudo firewall-cmd --list-all

1
2
3
4
5
6
7
8
9
10
11
12
# On 10.0.0.1 (Node 0)
echo 0 | sudo tee /etc/pgpool-II/pgpool_node_id

# On 10.0.0.2 (Node 1)
echo 1 | sudo tee /etc/pgpool-II/pgpool_node_id

# On 10.0.0.3 (Node 2)
echo 2 | sudo tee /etc/pgpool-II/pgpool_node_id

# On ALL nodes after setting the ID:
sudo chown postgres:postgres /etc/pgpool-II/pgpool_node_id
sudo chmod 644 /etc/pgpool-II/pgpool_node_id

1
2
3
4
5
6
7
sudo tee /etc/pgpool-II/pool_hba.conf << 'EOF'
# TYPE  DATABASE  USER       CIDR-ADDRESS  METHOD
local   all       all                      trust
host    all       postgres   0.0.0.0/0     trust
host    all       pgpool     0.0.0.0/0     scram-sha-256
host    all       app_user   0.0.0.0/0     scram-sha-256
EOF

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
sudo su - postgres

# Create decryption key file
cat > ~/.pgpoolkey << 'EOF'
YourEncryptionKey
EOF
chmod 600 ~/.pgpoolkey

# Encrypt pgpool user password
pg_enc -m -k ~/.pgpoolkey -u pgpool -p
# Enter: ChangeMe123!

# Encrypt application user password
pg_enc -m -k ~/.pgpoolkey -u app_user -p
# Enter: AppUserPass123!

# Verify both users appear
cat /etc/pgpool-II/pool_passwd
# Should show two AES-encrypted lines

exit

1
2
3
4
5
6
7
8
9
# Generate MD5 hash of the PCP password
# Replace 'ChangeMe123!' with your actual PCP password
echo -n 'ChangeMe123!' | md5sum | awk '{print $1}'
# Example output: a7eb3760e8253f69f17dcfa5e0c3d0eb

# Add to pcp.conf
sudo tee /etc/pgpool-II/pcp.conf << 'EOF'
pgpool:a7eb3760e8253f69f17dcfa5e0c3d0eb
EOF

1
2
3
4
5
6
sudo su - postgres
cat > ~/.pcppass << 'EOF'
*:*:pgpool:ChangeMe123!
EOF
chmod 600 ~/.pcppass
exit

1
sudo visudo

1
sudo systemctl edit pgpool.service

1
sudo systemctl daemon-reload

1
sudo cp /etc/pgpool-II/pgpool.conf /etc/pgpool-II/pgpool.conf.default

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
sudo tee /etc/pgpool-II/pgpool.conf << 'EOF'
# =============================================================
# Pgpool-II 4.5 Configuration — HA Cluster
# Generated for: 12 vCPU / 20GB RAM, 3-node watchdog
# =============================================================

# ---------------- CLUSTERING MODE ----------------
backend_clustering_mode = 'streaming_replication'

# ---------------- CONNECTIONS ----------------
listen_addresses = '*'
port = 9999
reserved_connections = 50
listen_backlog_multiplier = 2
serialize_accept = off
pcp_listen_addresses = '*'
pcp_port = 9898
unix_socket_directories = '/var/run/pgpool-II'
pcp_socket_dir = '/var/run/pgpool-II'
wd_ipc_socket_dir = '/var/run/pgpool-II'

# ---------------- CONNECTION POOLING ----------------
num_init_children = 1000
max_pool = 2
child_life_time = 5
minchild_max_connections = 1000
connection_life_time = 600
client_idle_limit = 5min
connection_cache = on
reset_query_list = 'ABORT; DISCARD ALL'

# ---------------- AUTHENTICATION ----------------
enable_pool_hba = on
pool_passwd = 'pool_passwd'
authentication_timeout = 1min

# ---------------- BACKEND (POSTGRESQL) NODES ----------------
backend_hostname0 = '10.0.0.1'
backend_port0 = 5432
backend_weight0 = 1
backend_data_directory0 = '/data/pgdata/17/data'
backend_flag0 = 'ALLOW_TO_FAILOVER'
backend_application_name0 = 'db-node-01'

backend_hostname1 = '10.0.0.2'
backend_port1 = 5432
backend_weight1 = 2
backend_data_directory1 = '/data/pgdata/17/data'
backend_flag1 = 'ALLOW_TO_FAILOVER'
backend_application_name1 = 'db-node-02'

backend_hostname2 = '10.0.0.3'
backend_port2 = 5432
backend_weight2 = 2
backend_data_directory2 = '/data/pgdata/17/data'
backend_flag2 = 'ALLOW_TO_FAILOVER'
backend_application_name2 = 'db-node-03'

# ---------------- REPLICATION MODE ----------------
replicate_select = off

# ---------------- LOGGING ----------------
log_destination = 'stderr'
logging_collector = on
log_hostname = on
log_connections = off
log_disconnections = off
log_pcp_processes = on
log_per_node_statement = off
log_statement = off
log_client_messages = off
log_standby_delay = 'if_over_threshold'
log_error_verbosity = default
log_min_messages = warning
log_directory = '/data/log/pgpool'
log_filename = 'pgpool-%Y-%m-%d.log'
log_truncate_on_rotation = on
log_rotation_age = 1d
log_rotation_size = 0
pid_file_name = '/var/run/pgpool-II/pgpool.pid'

# ---------------- LOAD BALANCING ----------------
load_balance_mode = on
ignore_leading_white_space = on
allow_sql_comments = off
disable_load_balance_on_write = 'transaction'
statement_level_load_balance = on
black_function_list = 'currval,lastval,nextval,setval'

# ---------------- STREAMING REPLICATION CHECK ----------------
sr_check_period = 10
sr_check_user = 'pgpool'
sr_check_password = ''
sr_check_database = 'postgres'
delay_threshold = 10000000

# ---------------- HEALTH CHECK ----------------
health_check_period = 10
health_check_timeout = 20
health_check_user = 'pgpool'
health_check_password = ''
health_check_database = 'postgres'
health_check_max_retries = 3
health_check_retry_delay = 5
connect_timeout = 10000

# ---------------- FAILOVER ----------------
failover_on_backend_error = on
detach_false_primary = off
search_primary_node_timeout = 5min
auto_failback = off
auto_failback_interval = 1min

# ---------------- WATCHDOG ----------------
use_watchdog = on
trusted_servers = '10.0.0.1,10.0.0.2,10.0.0.3'
hostname0 = '10.0.0.1'
wd_port0 = 9000
pgpool_port0 = 9999
hostname1 = '10.0.0.2'
wd_port1 = 9000
pgpool_port1 = 9999
hostname2 = '10.0.0.3'
wd_port2 = 9000
pgpool_port2 = 9999
wd_priority = 1

# ---------------- VIRTUAL IP ----------------
delegate_ip = '10.0.0.100'
if_cmd_path = '/sbin'
if_up_cmd = '/usr/bin/sudo /sbin/ip addr add $_IP_$/23 dev ens160 label ens160:0'
if_down_cmd = '/usr/bin/sudo /sbin/ip addr del $_IP_$/23 dev ens160'
arping_cmd = '/usr/bin/sudo /usr/sbin/arping -U $_IP_$ -w 1 -I ens160'

# ---------------- WATCHDOG BEHAVIOR ----------------
clear_memqcache_on_escalation = on
failover_when_quorum_exists = on
failover_require_consensus = on
allow_multiple_failover_requests_from_node = off
enable_consensus_with_half_votes = on

# ---------------- WATCHDOG LIFECHECK ----------------
wd_monitoring_interfaces_list = 'ens160'
wd_lifecheck_method = 'heartbeat'
wd_interval = 10
heartbeat_hostname0 = '10.0.0.1'
heartbeat_port0 = 9694
heartbeat_device0 = 'ens160'
heartbeat_hostname1 = '10.0.0.2'
heartbeat_port1 = 9694
heartbeat_device1 = 'ens160'
heartbeat_hostname2 = '10.0.0.3'
heartbeat_port2 = 9694
heartbeat_device2 = 'ens160'
wd_life_point = 3
wd_lifecheck_query = 'SELECT 1'
wd_lifecheck_dbname = 'template1'
wd_lifecheck_user = 'pgpool'
wd_lifecheck_password = ''

# ---------------- MEMORY QUERY CACHE (disabled) ----------------
memory_cache_enabled = off
memqcache_method = 'shmem'
memqcache_total_size = 2GB
memqcache_max_num_cache = 1000000
memqcache_expire = 3600
memqcache_auto_cache_invalidation = on
memqcache_oiddir = '/data/log/pgpool/oiddir'
EOF

1
2
3
4
5
6
# Start Pgpool (do this on each node, one at a time)
sudo systemctl start pgpool.service
sudo systemctl enable pgpool.service

# Check status
sudo systemctl status pgpool.service

1
2
3
4
5
6
7
8
9
10
11
12
13
# Show pool node status (run from any node or via VIP)
psql -h 10.0.0.100 -p 9999 -U pgpool postgres -c "SHOW POOL_NODES;"
# Password: ChangeMe123!

# Check watchdog status
sudo -u postgres pcp_watchdog_info -h 10.0.0.100 -U pgpool -p 9898 -v
# Password: ChangeMe123!

# Check which node holds the VIP
for ip in 10.0.0.1 10.0.0.2 10.0.0.3; do
  echo "=== $ip ==="
  ssh postgres@$ip "ip addr show ens160 | grep 10.0.0.100" 2>/dev/null || echo "VIP not here"
done

1
2
3
4
5
6
7
8
9
10
# Test connection via VIP
psql -h 10.0.0.100 -p 9999 -U app_user appdb \
  -c "SELECT current_database(), inet_server_addr(), inet_server_port();"
# Password: AppUserPass123!

# Run 5 times to verify load balancing hits different backends
for i in {1..5}; do
  psql -h 10.0.0.100 -p 9999 -U app_user appdb \
    -c "SELECT inet_server_addr();" -t 2>/dev/null | tr -d ' '
done

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# Node status
psql -h 10.0.0.100 -p 9999 -U pgpool postgres -c "SHOW POOL_NODES;"

# Active processes
psql -h 10.0.0.100 -p 9999 -U pgpool postgres -c "SHOW POOL_PROCESSES;"

# Connection pool state
psql -h 10.0.0.100 -p 9999 -U pgpool postgres -c "SHOW POOL_POOLS;"

# PCP node info
pcp_node_info -h 10.0.0.100 -U pgpool -p 9898 -n 0
pcp_node_info -h 10.0.0.100 -U pgpool -p 9898 -n 1
pcp_node_info -h 10.0.0.100 -U pgpool -p 9898 -n 2

# Replication lag (on primary)
psql -h 10.0.0.1 -U postgres -c \
  "SELECT client_addr, state, sent_lsn, replay_lsn, \
   (sent_lsn - replay_lsn) AS lag_bytes FROM pg_stat_replication;"

# Re-attach a detached node
pcp_attach_node -h 10.0.0.100 -U pgpool -n 1 -p 9898
pcp_attach_node -h 10.0.0.100 -U pgpool -n 2 -p 9898

1
2
3
4
5
6
pcp_watchdog_info -h 10.0.0.1 -U pgpool -p 9898 -v
sudo -u postgres sudo /sbin/ip addr show ens160

# Manual emergency VIP assignment on the primary node:
sudo /sbin/ip addr add 10.0.0.100/23 dev ens160 label ens160:0
sudo /usr/sbin/arping -U 10.0.0.100 -w 1 -I ens160

1
2
3
4
5
systemctl status postgresql-17
pcp_attach_node -h 10.0.0.100 -U pgpool -p 9898 -n 1

# If still failing:
sudo systemctl restart pgpool

1
2
3
4
5
6
7
sudo cat /etc/pgpool-II/pool_passwd

# Re-encrypt if missing:
sudo su - postgres
pg_enc -m -k ~/.pgpoolkey -u pgpool -p
pg_enc -m -k ~/.pgpoolkey -u app_user -p
exit

1
2
psql -h 10.0.0.1 -U postgres -c "SELECT pg_reload_conf();"
psql -h 10.0.0.1 -U postgres -c "SELECT * FROM pg_hba_file_rules;"

1
2
3
4
5
6
7
8
9
# Check Ubuntu version
lsb_release -a

# Check Python version
python3 --version

# Check Docker and Docker Compose
docker --version
docker-compose --version

1
2
mkdir -p ~/airflow
export AIRFLOW_HOME=~/airflow

1
2
echo 'export AIRFLOW_HOME=~/airflow' >> ~/.bashrc  # or ~/.zshrc for Zsh users
source ~/.bashrc

1
2
3
4
5
6
7
# Update package lists and install dependencies
sudo apt-get update
sudo apt-get install -y python3-pip python3-venv

# Create and activate a virtual environment
python3 -m venv ~/venv
source ~/venv/bin/activate

1
2
3
4
5
6
7
8
9
10
11
# Set version variables
export AIRFLOW_VERSION=2.10.1
export PYTHON_VERSION=$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)

# Install dependencies and Airflow
CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
pip install "psycopg2-binary==2.9.6"
pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"

# Verify installation
airflow version

1
pip install apache-airflow-providers-slack --constraint "${CONSTRAINT_URL}"

1
2
3
4
5
6
7
airflow users create \
  --username admin \
  --password your_secure_password \
  --firstname Admin \
  --lastname User \
  --role Admin \
  --email admin@example.com

1
2
sudo chown -R <YOUR-USERNAME>:<YOUR-USERNAME> ~/airflow
sudo chmod -R 755 ~/airflow

1
2
3
4
5
6
7
8
9
10
11
# Start the service
sudo systemctl start airflow

# Stop the service
sudo systemctl stop airflow

# Restart the service
sudo systemctl restart airflow

# View real-time logs
journalctl -u airflow -f

1
2
mkdir -p ~/.pipelinewise
sudo chown -R <YOUR-USERNAME>:<YOUR-USERNAME> ~/.pipelinewise

1
plw import --dir /path/to/warehouse_configs/

1
2
3
4
# Clone the repository and run the installation script
git clone <your-git-repository-url>
cd cluster-doc
bash install-k8s-1.29.sh

1
2
3
4
5
6
7
8
9
10
11
12
# Initialize Kubernetes with a specific pod network CIDR
sudo kubeadm init --pod-network-cidr=<REDACTED_POD_NETWORK_CIDR>

# Download and install Cilium CLI
wget https://github.com/cilium/cilium-cli/releases/latest/download/cilium-linux-amd64.tar.gz
sudo tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin

# Install Cilium
cilium install

# Monitor Cilium status
watch 'cilium status'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Clone the Rook repository and checkout the specific version
git clone --single-branch --branch v1.14.8 https://github.com/rook/rook.git
cd rook/deploy/examples

# Deploy Rook Ceph components
kubectl create -f crds.yaml -f common.yaml -f operator.yaml
sleep 5
kubectl create -f cluster.yaml

# Create the Rook Ceph storage class
kubectl apply -f csi/rbd/storageclass.yaml

# Patch the storage class to make it the default
kubectl patch storageclass rook-ceph-block -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

1
2
3
4
5
6
7
8
# Clone the PostgreSQL Operator examples repository
git clone https://github.com/CrunchyData/postgres-operator-examples.git
cd postgres-operator-examples/

# Install the Crunchy PostgreSQL Operator
kubectl apply -k kustomize/install/namespace
kubectl apply --server-side -k kustomize/install/default
kubectl apply -k kustomize/postgres

1
2
3
4
# Download and install the kubectl-pgo client
wget https://github.com/CrunchyData/postgres-operator-client/releases/download/v0.4.2/kubectl-pgo-linux-amd64
sudo mv kubectl-pgo-linux-amd64 /usr/local/bin/kubectl-pgo
sudo chmod +x /usr/local/bin/kubectl-pgo

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
# Create the db-ns namespace
kubectl create ns db-ns

# Create a directory for your cluster configuration
mkdir mydatabase-k8s/

# Save the following YAML configuration to a file named mydatabase-13-demo.yaml
cat <<EOF > mydatabase-13-demo.yaml
apiVersion: postgres-operator.crunchydata.com/v1beta1
kind: PostgresCluster
metadata:
  name: mydatabase
  namespace: db-ns
  labels:
    pg-cluster: mydatabase
    pgo-version: 5.6.7
spec:
  image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.8-1
  postgresVersion: 13
  instances:
  - name: mydatabase
    replicas: 1
    dataVolumeClaimSpec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 5Gi
      storageClassName: rook-ceph-block
    resources:
      requests:
        cpu: "2"
        memory: "4Gi"
      limits:
        cpu: "2"
        memory: "4Gi"
  backups:
    pgbackrest:
      image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.40-1
      repos:
      - name: repo1
        volume:
          volumeClaimSpec:
            accessModes: ["ReadWriteOnce"]
            resources:
              requests:
                storage: 5Gi
            storageClassName: rook-ceph-block
  monitoring:
    pgmonitor:
      exporter:
        image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-15.7-1
  users:
  - name: mydatabaseuser
    databases: [mydb]
  patroni:
    dynamicConfiguration:
      postgresql:
        parameters:
          max_connections: "600"
          shared_buffers: "4GB"
          effective_cache_size: "6GB"
          maintenance_work_mem: "2GB"
          checkpoint_completion_target: "0.9"
          wal_buffers: "16MB"
          default_statistics_target: "100"
          random_page_cost: "1.1"
          effective_io_concurrency: "200"
          min_wal_size: "1GB"
          max_wal_size: "4GB"
          max_worker_processes: "8"
          max_parallel_workers_per_gather: "4"
          max_parallel_workers: "8"
          max_parallel_maintenance_workers: "4"
          wal_keep_size: "2048MB"
          max_standby_archive_delay: "-1"
          hot_standby: "on"
EOF

# Apply the YAML configuration to create the PostgreSQL cluster
kubectl apply -f mydatabase-13-demo.yaml

1
kubectl exec -it -n db-ns -c database $(kubectl get pods -n db-ns --selector='postgres-operator.crunchydata.com/cluster=mydatabase,postgres-operator.crunchydata.com/role=master' -o jsonpath='{.items[0].metadata.name}') -- pg_dump -s -U postgres -d mydb -f /tmp/mydatabase_schema_bkp_old.sql

1
kubectl exec -it -n db-ns -c database $NEW_MASTER_POD -- psql -U postgres -d mydb -f /tmp/mydatabase_schema_bkp_old.sql