Home / linux / kubenetes / install

Kubenetes install

Install Overview

Configuration Linux kernel settings for Kubernetes networking

These settings ensure that bridged network traffic is processed by Linux iptables firewall rules and enable IP forwarding:

# assuming ubuntu X86_64
# setting up container runtime prereq
    cat <<- EOF | sudo tee /etc/modules-load.d/containerd.conf
    overlay
    br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<- EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv6.ip_forward                 = 1
EOF

# Apply sysctl params without reboot
sudo sysctl --system

Installing CRI and Tools

# 1) https://github.com/sandervanvugt/cka setup-container.sh

CONTAINERD_VERSION=$(curl -s https://api.github.com/repos/containerd/containerd/releases/latest | jq -r '.tag_name')
CONTAINERD_VERSION=${CONTAINERD_VERSION#v}

wget https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-${PLATFORM}.tar.gz

sudo tar xvf containerd-${CONTAINERD_VERSION}-linux-${PLATFORM}.tar.gz -C /usr/local

sudo mkdir -p /etc/containerd

cat <<- TOML | sudo tee /etc/containerd/config.toml
version = 2
[plugins]
  [plugins."io.containerd.grpc.v1.cri"]
    [plugins."io.containerd.grpc.v1.cri".containerd]
      discard_unpacked_layers = true
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
TOML

wget https://github.com/opencontainers/runc/releases/download/${RUNC_VERSION}/runc.${PLATFORM}
sudo install -m 755 runc.${PLATFORM} /usr/local/sbin/runc

wget https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
sudo mv containerd.service /usr/lib/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now containerd

# verify containerd service are running
sudo systemctl status containerd

sudo ln -s /etc/apparmor.d/runc /etc/apparmor.d/disable/
sudo apparmor_parser -R /etc/apparmor.d/runc

# 2) https://github.com/sandervanvugt/cka/blob/master/setup-kubetools-previousversion.sh
KUBEVERSION=$(curl -s https://api.github.com/repos/kubernetes/kubernetes/releases/latest | jq -r '.tag_name')
KUBEVERSION=${KUBEVERSION%.*}

# setting previous version
VERSION=${KUBEVERSION#*.}
PREVIOUSVERSION=$(( VERSION - 1 ))
PREVIOUSVERSION=v1.${PREVIOUSVERSION}

sudo apt-get update && sudo apt-get install -y apt-transport-https curl

curl -fsSL https://pkgs.k8s.io/core:/stable:/${PREVIOUSVERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${PREVIOUSVERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl
sudo swapoff -a

sudo sed -i 's/\/swap/#\/swap/' /etc/fstab

sudo crictl config --set \
    runtime-endpoint=unix:///run/containerd/containerd.sock

Using kubeadm init

# `kubeadm init` command is used with root privileges on the control node only
# params:
#     --node-name: specifies the name of the Kubernetes node
#     --service-dns-domain: defines a DNS domain other than cluster.local
#     --apiserver-advertise-address: the IP address on which the API server is listening
#     --config: the name of a configuration file used for additional configuration
#     --dry-run: performs a dry run before actually installing for real
#     --pod-network-cidr: sets the CIDR used for the Pod network
#     --service-cidr: sets the service CIDR to something other than 10.96.0.0/12

# Exec Steps:
# - preflight: ensures all conditions are met and core container images are downloaded
# - certs: a self-signed Kubernetes CA is generated, and related certificates are created for apiserver, etcd, and proxy
# - kubeconfig: configuration files are generated for core Kubernetes services
# - kubelete-start: the kubelete is started
# - control-plane: static Pods manifests are created and started for apiserver, controller manager, and scheduler
# - etcd: static Pod manifests are created and started for etcd
# - upload-config: ConfigMaps are created for cluster configuration and kubelet component config
# - upload-certs: uploads all certificates to /etc/kubenetes/pki
# - mark-control-plane: marks the node as control plane
# - bootstrap-token: generates the token that can be used to join other nodes
# - kubelet-finalize: finalizes kubelet settings
# - add-on: installs coredns and kube-proxy add-ons

# If `kubeadm init` fails, use `kubeadm reset` to remove components that were installed and configured

# wirte all configuration parameters to config.yaml
sudo kubeadm config print init-defaults > config.yaml

# edit config.yaml as desired, at least the following parameters:
# - localAPIEndpoint.advertiseAddress: a valid IP address on which the apiserver is available
# - nodeRegistration.namef: the name of the apiserver node
sudo kubeadmin init --config config.yaml

Configuring the Client

After cluster initialzation, the /etc/kubernetes/admin.conf file is copied to ~/.kube/config to provide admin access to the Kubernetes cluster

# verify config success
kubectl get all
# view the context
kubectl config view
# set a differ context
kubectl set-context
# use a context
kubectl use-context

# define other clusters
kubectl config --kubeconfig=~/.kube/config set-cluster devcluster --server=https://192.168.29.120 --certificate-authority=clusterca.crt

# create namespace
kubectl create ns

# define user
kubectl config --kubeconfig=~/.kube/config set-credentials anna --client-certificate=anna.crt --client.key=anna.key
# use cluster
kubectl set-context devluster --cluster=devcluster --namespace-devspace --user=anna

Setting up Node Networking

Different types of network communication are used in Kubernetes

kubectl get pods -n kube-system
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
kubectl get pods -n kube-system -w

Adding Nodes to the Cluster

Before adding nodes, make sure the node name is set to what you want to use

Nodes are added by name, if the node name changes, the node needs to be removed and added again

# when the Kubernetes cluster is initialized, a join token is generated
sudo kubeadm join
# in case the join token is lost or expired, use
sudo kubeadm token create --print-join-command

Deployment Tools

Docker Desktop

Including a local Kubernetes cluster for Docker users. Enable kubenetes in settings curl.exe -LO "https://dl.k8s.io/release/v1.29.2/bin/windows/amd64/kubectl.exe" & add path to env kubectl config current-context // view current kubenetes cluster

Minikube

Single- and multi-node local Kubernetes cluster, recommended for a learning environment deployed on a single host.

minikube command reference

## customize the VM isolation driver, container runtime, profile name
minikube start --kubernetes-version=v1.23.3 \
--driver=podman --profile minipod

minikube start --nodes=2 --kubernetes-version=v1.24.4 \
--driver=docker --profile doubledocker

minikube start --driver=virtualbox --nodes=3 --disk-size=10g \
--cpus=2 --memory=4g --kubernetes-version=v1.25.1 --cni=calico \
--container-runtime=cri-o -p multivbox

minikube start --driver=docker --cpus=6 --memory=8g \
--kubernetes-version="1.24.4" -p largedock

minikube start --driver=virtualbox -n 3 --container-runtime=containerd \
--cni=calico -p minibox

## Completion is a helpful post installation configuration to enable the minikube command to respond to typical auto-completion mechanisms
sudo apt install bash-completion

source /etc/bash_completion

source <(minikube completion bash)

# If needed, also run the following command:s
minikube completion bash

linux install

Creating a Kubernetes (K8s) cluster with two computers involves several steps. Below is a general procedure to set up a basic Kubernetes cluster using two nodes. In this example, we'll refer to the two computers as Node1 and Node2.

Prerequisites:

Procedure:

  1. Install kubectl: Install kubectl on both Node1 and Node2. Kubectl is the command-line tool for interacting with a Kubernetes cluster.
   # On Ubuntu
   sudo apt-get update && sudo apt-get install -y kubectl

   # On CentOS
   sudo yum install -y kubectl
  1. Install kubeadm, kubelet, and kubectl: Install Kubernetes components on both Node1 and Node2.
   # On both nodes
   sudo apt-get update && sudo apt-get install -y apt-transport-https curl
   sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
   sudo echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
   sudo apt-get update
   sudo apt-get install -y kubelet kubeadm kubectl
   sudo apt-mark hold kubelet kubeadm kubectl  # Avoid automatic updates
  1. Initialize the master node (Node1): On Node1, run the following command to initialize the Kubernetes master.
   sudo kubeadm init --pod-network-cidr=192.168.0.0/16

After the initialization is complete, follow the on-screen instructions to copy the kubeadm join command.

  1. Configure kubectl on the master node: On Node1, set up the kubeconfig file for kubectl.
   mkdir -p $HOME/.kube
   sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
   sudo chown $(id -u):$(id -g) $HOME/.kube/config
  1. Install a network plugin: Choose a network plugin that suits your needs. In this example, we'll use Calico.
   kubectl apply -f https://docs.projectcalico.org/v3.18/manifests/calico.yaml
  1. Join the worker node (Node2) to the cluster: On Node2, run the kubeadm join command obtained from the master node initialization step.
   sudo kubeadm join <master-node-ip>:<master-node-port> --token <token> --discovery-token-ca-cert-hash sha256:<hash>

Replace <master-node-ip>, <master-node-port>, <token>, and <hash> with the values specific to your setup.

  1. Verify the cluster: On Node1, run the following command to verify that both nodes are part of the cluster.
   kubectl get nodes

Both nodes should be in the Ready state.

Congratulations! You now have a basic Kubernetes cluster with two computers. Keep in mind that this is a simplified setup, and in a production environment, you may need to consider additional configurations and security measures.

Build a kubenetes cluster

# master 1 VM, slave 1 VM, slave 2 VM
# turn off swap
sudo swapoff -a
# edit /etc/fstab comment out swap line
sudo vim /etc/fstab
# install docker
sudo apt install docker.io -y
sudo apt install apt-transport-https curl -y
# add repository key
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add
sudo apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main"
sudo apt update
sudo apt install kubeadm kubelet kubectl kubenetes-cni -y

# exec following line only on master node
# there will be `kubeadmin join link after command finishes`, copy and paste
# to the other slave VM
sudo kubeadm init

# on slate VM, execute
kubeadm join <ip:port> --token <string> --discovery-token-ca-cert-hash sha:256:<string>

# on master 1
kubectl get nodes
# The connection to the server localhost:8080 was refused - did you specify the right host or port?
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get nodes
# NAME      STATUS   ROLES    AGE      VERSION
# master    NotReady control-plane  9m21s v1.24.3
# slave1    NotReady <none>   2m1s     v1.24.3
# slave2    NotReady <none>   114s     v1.24.3

# deploy network config
kubectl apply -f https://docs.projectcalico.org/mainifests/calico.yaml
# NAME      STATUS   ROLES    AGE      VERSION
# master    Ready    control-plane  22m v1.24.3
# slave1    Ready    <none>   14m     v1.24.3
# slave2    Ready    <none>   14m     v1.24.3

## ingress-nginx
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.10.0/deploy/static/provider/cloud/deploy.yaml

kubectl -n ingress-nginx get pod -o yaml
kubectl get service ingress-nginx-controller --namespace=ingress-nginx

## https://kubernetes.github.io/ingress-nginx/deploy/

国内环境 Kubernetes 集群搭建指南

Linux node

基础环境设置

# 更新系统
sudo apt update && sudo apt upgrade -y

# 关闭防火墙
sudo ufw disable

# 关闭swap分区
sudo swapoff -a
sudo sed -i '/swap/d' /etc/fstab

# 设置系统参数
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

sudo sysctl --system

安装容器运行时（Containerd）

# 添加Docker源
sudo apt install -y ca-certificates curl gnupg
sudo install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

# 安装Containerd
sudo apt update
sudo apt install -y containerd.io

# 配置Containerd使用systemd cgroup驱动
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

# 修改配置使用systemd cgroup和国内镜像
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
sudo sed -i 's|registry.k8s.io|registry.aliyuncs.com/google_containers|g' /etc/containerd/config.toml

# 添加Docker中国镜像
cat <<EOF | sudo tee -a /etc/containerd/config.toml
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
    endpoint = ["https://registry.docker-cn.com"]
EOF

# 重启Containerd
sudo systemctl restart containerd
sudo systemctl enable containerd

安装 Kubernetes 组件

# 添加Kubernetes源
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

# 安装Kubeadm, Kubelet和Kubectl
sudo apt update
sudo apt install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

# 启动Kubelet
sudo systemctl enable kubelet
sudo systemctl start kubelet

Windows 节点配置

启用 Hyper-V 和 WSL2

# 以管理员身份运行PowerShell
Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All
dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart
dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart

# 下载并安装WSL2 Linux内核更新包
# https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi

# 设置WSL2为默认版本
wsl --set-default-version 2

# 安装Ubuntu 20.04 LTS from Microsoft Store

在 WSL2 中安装 Kubernetes 组件

# 在WSL2 Ubuntu中执行
# 重复Linux节点的配置步骤
# 注意：WSL2环境中kubelet可能需要特殊配置
echo "KUBELET_EXTRA_ARGS=--node-ip=$(hostname -I | awk '{print $1}')" | sudo tee /etc/default/kubelet

使用 Kuboard-Spray 部署集群

准备 Kuboard-Spray 环境

# 在Linux节点上执行
mkdir -p ~/kuboard-spray && cd ~/kuboard-spray
docker run -it --rm -v `pwd`:/data eipwork/kuboard-spray:latest-amd64 init

# 修改inventory.ini配置文件
vim inventory.ini

# 示例配置
[all]
master1 ansible_host=192.168.1.100 ip=192.168.1.100 etcd_member_name=etcd1
worker1 ansible_host=192.168.1.101 ip=192.168.1.101

[kube-master]
master1

[kube-worker]
worker1

[etcd]
master1

[k8s-cluster:children]
kube-master
kube-worker

配置镜像加速

# 修改group_vars/all.yml
vim group_vars/all.yml

# 添加以下内容
docker_registry_mirrors:
  - https://registry.docker-cn.com
  - https://hub-mirror.c.163.com
  - https://mirror.baidubce.com

image_repository: registry.aliyuncs.com/google_containers

执行部署

# 执行预检查
docker run -it --rm -v `pwd`:/data eipwork/kuboard-spray:latest-amd64 precheck

# 部署集群
docker run -it --rm -v `pwd`:/data eipwork/kuboard-spray:latest-amd64 apply

安装 Kuboard 界面

# 在部署完成后执行
kubectl apply -f https://addons.kuboard.cn/kuboard/kuboard-v3.yaml

# 查看Kuboard服务
kubectl get pods,svc -n kuboard

访问 Kuboard

# 查看NodePort
kubectl get svc kuboard -n kuboard

# 访问地址
# http://<节点IP>:<NodePort>
# admin/Kuboard123

# 如果遇到镜像拉取问题，可以手动拉取并标记：
# 示例：拉取kube-apiserver
docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.27.4
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.27.4 registry.k8s.io/kube-apiserver:v1.27.4
Page Source