Kubernetes learn

# debian 13 x86-64 kernel 6.13.9
sudo mount -o remount,size=16G,noatime /run
sudo apt install kubectl
mkdir /tmp/kubernetes
cd /tmp/kubernetes
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
curl -o debian.tar.xz -L https://cloud.debian.org/images/cloud/trixie/daily/20250421-2089/debian-13-generic-amd64-daily-20250421-2089.tar.xz # genericcloud variant is broken currently
tar -xf debian.tar.xz
sudo losetup --find --show --partscan disk.raw
mkdir disk
sudo mount /dev/loop0p1 disk
sudo chroot disk
passwd # root root
sudo umount disk
sudo losetup --detach /dev/loop0
qemu-img convert -p -f raw -O qcow2 -c -o compression_type=zstd,preallocation=off disk.raw debian.qcow2
chmod 444 debian.qcow2
rm -rf disk.raw disk debian.tar.xz

# 1 master node, 2 worker nodes. 1 cpu for each node

sudo ip link add br0 type bridge
sudo ip link set dev br0 up
sudo ip addr add 127.0.0.1/24 dev br0

sudo nft add table ip nat # https://wiki.nftables.org/wiki-nftables/index.php/Quick_reference-nftables_in_10_minutes
sudo nft add chain ip nat postrouting { type nat hook postrouting priority 100 \; }
sudo nft add rule ip nat postrouting ip saddr 172.17.7.0/24 oifname eno1 masquerade

n=11 # 11 21 22 23 ... must be 2 digits

# sudo ip tuntap add mode tap tap$n
# sudo ip link set dev tap$n master br0
# sudo ip link set dev tap$n up
# qemu-img create -q -F qcow2 -b debian.qcow2 -f qcow2 n$n.qcow2
# qemu-system-x86_64 -name n$n,process=n$n -machine q35,accel=kvm -bios /usr/share/ovmf/OVMF.fd -cpu host -smp 2 -m 2G -hda n$n.qcow2 --nographic -nic tap,mac=52:54:00:12:34:$n,ifname=tap$n,script=no,downscript=no
# sudo ip link del tap$n

# sysctl -w net.ipv4.ip_forward=1
ip link set dev enp0s2 up
ip addr add 172.17.7.$n/24 brd + dev enp0s2
ip route add default via 127.0.0.1 dev enp0s2
rm /etc/resolv.conf
echo "nameserver 223.5.5.5" > /etc/resolv.conf

printf "Types: deb\nURIs: http://mirrors.nju.edu.cn/debian\nSuites: trixie trixie-updates trixie-backports\nComponents: main contrib non-free non-free-firmware\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n\nTypes: deb\nURIs: http://mirrors.nju.edu.cn/debian-security\nSuites: trixie-security\nComponents: main contrib non-free non-free-firmware\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n\n# > https://help.mirrorz.org/debian/\n# http://mirrors.bfsu.edu.cn/debian # and -security\n# http://mirrors.ustc.edu.cn/debian # and -security\n" > /etc/apt/sources.list.d/debian.sources
apt-get update
apt-get install -y --no-install-recommends gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
printf "Types: deb\nURIs: http://mirrors.nju.edu.cn/kubernetes/core:/stable:/v1.32/deb/\nSuites: /\nSigned-By: /etc/apt/keyrings/kubernetes-apt-keyring.gpg\n\nTypes: deb\nURIs: http://mirrors.nju.edu.cn/kubernetes/addons:/cri-o:/stable:/v1.32/deb/\nSuites: /\nSigned-By: /etc/apt/keyrings/kubernetes-apt-keyring.gpg\n\n" > /etc/apt/sources.list.d/kubernetes.sources
apt-get update
apt-get install -y kubelet kubeadm kubectl cri-o
apt-cache policy kubectl # confirm that you installed from kubernetes apt repo

apt purge --autoremove -y systemd-timesyncd unattended-upgrades cloud-init apparmor openssh-client openssh-server reportbug man-db manpages
update-locale LANG=C.UTF-8 LC_ALL=C.UTF-8

mkdir /etc/systemd/system/docker.service.d
cat <<EOF > /etc/systemd/system/docker.service.d/proxy.conf
[Service]
Environment="HTTP_PROXY=socks5h://127.0.0.1:9091"
Environment="HTTPS_PROXY=socks5h://127.0.0.1:9091"
Environment="ALL_PROXY=socks5h://127.0.0.1:9091"
Environment="NO_PROXY=localhost"
EOF
systemctl daemon-reload
systemctl restart cri-o
swapoff -a
modprobe br_netfilter # https://github.com/cri-o/packaging/blob/main/README.md#bootstrap-a-cluster-1
sysctl -w net.ipv4.ip_forward=1
kubeadm init

modprobe br_netfilter

curl  -o minikube -L https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-amd64
chmod +x minikube
mv minikube /usr/local/bin/minikube


# curl --proxy socks5h://127.0.0.1:9091 -L https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.33.0/crictl-v1.33.0-linux-amd64.tar.gz

export HTTP_PROXY=socks5h://127.0.0.1:9091
export HTTPS_PROXY=socks5h://127.0.0.1:9091
export ALL_PROXY=socks5h://127.0.0.1:9091
export NO_PROXY=localhost

apt-get install -y conntrack

minikube start --driver=none

# echo -e "\nPermitRootLogin yes\nPermitEmptyPasswords yes\nPasswordAuthentication yes\n" >>/etc/ssh/sshd_config

# ip addr add 172.17.7.21/24 brd + dev br-lan

# ip addr add 172.17.7.21/24 dev tap21
# ip route add default via 127.0.0.1 dev br-lan