#!/usr/bin/env bash

set -e

source "${KUBEVIRTCI_PATH}/../hack/detect_cri.sh"
export CRI_BIN=${CRI_BIN:-$(detect_cri)}

export KIND_EXPERIMENTAL_PROVIDER=${CRI_BIN}
CONFIG_WORKER_CPU_MANAGER=${CONFIG_WORKER_CPU_MANAGER:-false}
# only setup ipFamily when the environmental variable is not empty
# avaliable value: ipv4, ipv6, dual
IPFAMILY=${IPFAMILY}

# setup the port mapping for kind cluster, this is needed for some e2e tests
# KIND_PORT_MAPPING=cluster_port:host_port e.g. KIND_PORT_MAPPING=30001:30002
# only one port mapping allowed
KIND_PORT_MAPPING=${KIND_PORT_MAPPING}

# check CPU arch
PLATFORM=$(uname -m)
case ${PLATFORM} in
x86_64* | i?86_64* | amd64*)
    ARCH="amd64"
    ;;
aarch64* | arm64*)
    ARCH="arm64"
    ;;
*)
    echo "invalid Arch, only support x86_64 and aarch64"
    exit 1
    ;;
esac

NODE_CMD="${CRI_BIN} exec -it -d "
export KIND_MANIFESTS_DIR="${KUBEVIRTCI_PATH}/cluster/kind/manifests"
export KIND_NODE_CLI="${CRI_BIN} exec -it "
export KUBEVIRTCI_PATH
export KUBEVIRTCI_CONFIG_PATH
KIND_DEFAULT_NETWORK="kind"

KUBECTL="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl --kubeconfig=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig"

REGISTRY_NAME=${CLUSTER_NAME}-registry

MASTER_NODES_PATTERN="control-plane"
WORKER_NODES_PATTERN="worker"

KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY=${KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY:-"true"}
ETCD_IN_MEMORY_DATA_DIR="/tmp/kind-cluster-etcd"

function _wait_kind_up {
    echo "Waiting for kind to be ready ..."
    if [[ $KUBEVIRT_PROVIDER =~ kind-.*1\.1.* ]]; then
        selector="master"
    else
        selector="control-plane"
    fi
    while [ -z "$(${CRI_BIN} exec --privileged ${CLUSTER_NAME}-control-plane kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes --selector=node-role.kubernetes.io/${selector} -o=jsonpath='{.items..status.conditions[-1:].status}' | grep True)" ]; do
        echo "Waiting for kind to be ready ..."
        sleep 10
    done
    echo "Waiting for dns to be ready ..."
    _kubectl wait -n kube-system --timeout=12m --for=condition=Ready -l k8s-app=kube-dns pods
}

function _wait_containers_ready {
    echo "Waiting for all containers to become ready ..."
    _kubectl wait --for=condition=Ready pod --all -n kube-system --timeout 12m
}

function _fetch_kind() {
    KIND="${KUBEVIRTCI_CONFIG_PATH}"/"$KUBEVIRT_PROVIDER"/.kind
    current_kind_version=$($KIND --version |& awk '{print $3}')
    if [[ $current_kind_version != $KIND_VERSION ]]; then
        echo "Downloading kind v$KIND_VERSION"
        curl -LSs https://github.com/kubernetes-sigs/kind/releases/download/v$KIND_VERSION/kind-linux-${ARCH} -o "$KIND"
        chmod +x "$KIND"
    fi
}

function _configure-insecure-registry-and-reload() {
    local cmd_context="${1}" # context to run command e.g. sudo, docker exec
    ${cmd_context} "$(_insecure-registry-config-cmd)"
    ${cmd_context} "$(_reload-containerd-daemon-cmd)"
}

function _reload-containerd-daemon-cmd() {
    echo "systemctl restart containerd"
}

function _insecure-registry-config-cmd() {
    echo '
    mkdir -p /etc/containerd/certs.d/registry:5000
    cat > /etc/containerd/certs.d/registry:5000/hosts.toml <<EOF
server = "http://registry:5000"

[host."http://registry:5000"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
EOF
    '
}
# this works since the nodes use the same names as containers
function _ssh_into_node() {
    if [[ $2 != "" ]]; then
        ${CRI_BIN} exec "$@"
    else
        ${CRI_BIN} exec -it "$1" bash
    fi
}

function _run_registry() {
    local -r network=${1}

    until [ -z "$($CRI_BIN ps -a | grep $REGISTRY_NAME)" ]; do
        ${CRI_BIN} stop $REGISTRY_NAME || true
        ${CRI_BIN} rm $REGISTRY_NAME || true
        sleep 5
    done
    ${CRI_BIN} run -d --network=${network} -p $HOST_PORT:5000  --restart=always --name $REGISTRY_NAME quay.io/kubevirtci/library-registry:2.7.1

}

function _configure_registry_on_node() {
    local -r node=${1}
    local -r network=${2}

    _configure-insecure-registry-and-reload "${NODE_CMD} ${node} bash -c"
    ${NODE_CMD} ${node} sh -c "echo $(${CRI_BIN} inspect --format "{{.NetworkSettings.Networks.${network}.IPAddress }}" $REGISTRY_NAME)'\t'registry >> /etc/hosts"
}

function _install_cnis {
    _install_cni_plugins
}

function _install_cni_plugins {
    local CNI_VERSION="v0.8.5"
    local CNI_ARCHIVE="cni-plugins-linux-${ARCH}-$CNI_VERSION.tgz"
    local CNI_URL="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION/$CNI_ARCHIVE"
    if [ ! -f ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/$CNI_ARCHIVE ]; then
        echo "Downloading $CNI_ARCHIVE"
        curl -sSL -o ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/$CNI_ARCHIVE $CNI_URL
    fi

    for node in $(_get_nodes | awk '{print $1}'); do
        ${CRI_BIN} cp "${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/$CNI_ARCHIVE" $node:/
        ${CRI_BIN} exec $node /bin/sh -c "tar xf $CNI_ARCHIVE -C /opt/cni/bin"
    done
}

function prepare_config() {
    BASE_PATH=${KUBEVIRTCI_CONFIG_PATH:-$PWD}
    cat >$BASE_PATH/$KUBEVIRT_PROVIDER/config-provider-$KUBEVIRT_PROVIDER.sh <<EOF
master_ip="127.0.0.1"
kubeconfig=${BASE_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
kubectl=${BASE_PATH}/$KUBEVIRT_PROVIDER/.kubectl
docker_prefix=localhost:${HOST_PORT}/kubevirt
manifest_docker_prefix=registry:5000/kubevirt
EOF
}

function _configure_network() {
    # modprobe is present inside kind container but may be missing in the
    # environment running this script, so load the module from inside kind
    ${NODE_CMD} $1 modprobe br_netfilter
    for knob in arp ip ip6; do
        ${NODE_CMD} $1 sysctl -w sys.net.bridge.bridge-nf-call-${knob}tables=1
    done
}

function _get_nodes() {
    _kubectl get nodes --no-headers
}

function _get_pods() {
    _kubectl get pods --all-namespaces --no-headers
}

function _fix_node_labels() {
    # Due to inconsistent labels and taints state in multi-nodes clusters,
    # it is nessecery to remove taint NoSchedule and set role labels manualy:
    #   Control-plane nodes might lack 'scheduable=true' label and have NoScheduable taint.
    #   Worker nodes might lack worker role label.
    master_nodes=$(_get_nodes | grep -i $MASTER_NODES_PATTERN | awk '{print $1}')
    for node in ${master_nodes[@]}; do
        # removing NoSchedule taint if is there
        if _kubectl taint nodes $node node-role.kubernetes.io/master:NoSchedule- || _kubectl taint nodes $node node-role.kubernetes.io/control-plane:NoSchedule-; then
            _kubectl label node $node kubevirt.io/schedulable=true
        fi
    done

    worker_nodes=$(_get_nodes | grep -i $WORKER_NODES_PATTERN | awk '{print $1}')
    for node in ${worker_nodes[@]}; do
        _kubectl label node $node kubevirt.io/schedulable=true
        _kubectl label node $node node-role.kubernetes.io/worker=""
    done
}

function _dump_kind_node_logs() {
    echo "=== Dumping kind node container logs for debugging ==="
    for container in $(${CRI_BIN} ps -a --filter "label=io.x-k8s.kind.cluster=${CLUSTER_NAME}" --format '{{.Names}}'); do
        echo "--- Container logs: ${container} ---"
        ${CRI_BIN} logs "${container}" 2>&1 || true
        echo "--- End container logs: ${container} ---"
    done
    echo "=== End kind node container logs ==="
}

function setup_kind() {
    $KIND -v 9 create cluster --retain --name=${CLUSTER_NAME} --config=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml --image=$KIND_NODE_IMAGE --kubeconfig=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig \
        || ( _dump_kind_node_logs; \
        $KIND -v 9 delete cluster --name=${CLUSTER_NAME} \
        && $KIND -v 9 create cluster --retain --name=${CLUSTER_NAME} --config=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml --image=$KIND_NODE_IMAGE --kubeconfig=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig )

    if ${CRI_BIN} exec ${CLUSTER_NAME}-control-plane ls /usr/bin/kubectl > /dev/null; then
        kubectl_path=/usr/bin/kubectl
    elif ${CRI_BIN} exec ${CLUSTER_NAME}-control-plane ls /bin/kubectl > /dev/null; then
        kubectl_path=/bin/kubectl
    else
        echo "Error: kubectl not found on node, exiting"
        exit 1
    fi

    ${CRI_BIN} cp ${CLUSTER_NAME}-control-plane:$kubectl_path ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl

    chmod u+x ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl

    if [ $KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY == "true" ]; then
        for node in $(_get_nodes | awk '{print $1}' | grep control-plane); do
            echo "[$node] Checking KIND cluster etcd data is mounted to RAM: $ETCD_IN_MEMORY_DATA_DIR"
            ${CRI_BIN} exec $node df -h $(dirname $ETCD_IN_MEMORY_DATA_DIR) | grep -P '(tmpfs|ramfs)'
            [ $(echo $?) != 0 ] && echo "[$node] etcd data directory is not mounted to RAM" && return 1

            ${CRI_BIN} exec $node du -h $ETCD_IN_MEMORY_DATA_DIR
            [ $(echo $?) != 0 ] && echo "[$node] Failed to check etcd data directory" && return 1
        done
    fi

    _install_cnis

    _wait_kind_up
    _kubectl cluster-info
    _fix_node_labels

    until _get_nodes
    do
        echo "Waiting for all nodes to become ready ..."
        sleep 10
    done

    # wait until k8s pods are running
    while [ -n "$(_get_pods | grep -v Running)" ]; do
        echo "Waiting for all pods to enter the Running state ..."
        _get_pods | >&2 grep -v Running || true
        sleep 10
    done

    _wait_containers_ready
    _run_registry "$KIND_DEFAULT_NETWORK"

    for node in $(_get_nodes | awk '{print $1}'); do
        _configure_registry_on_node "$node" "$KIND_DEFAULT_NETWORK"
        _configure_network "$node"
    done
    prepare_config

    if [[ $KUBEVIRT_DEPLOY_CDI == "true" ]]; then
       KUBEVIRT_CUSTOM_CDI_VERSION=${KUBEVIRT_CUSTOM_CDI_VERSION:-"v1.63.0"}
      _kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/"$KUBEVIRT_CUSTOM_CDI_VERSION"/cdi-operator.yaml
      _kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/"$KUBEVIRT_CUSTOM_CDI_VERSION"/cdi-cr.yaml
    fi

}

function _add_extra_mounts() {
  cat <<EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
  extraMounts:
  - containerPath: /var/log/audit
    hostPath: /var/log/audit
    readOnly: true
EOF

    if [[ "$KUBEVIRT_PROVIDER" =~ sriov.* || "$KUBEVIRT_PROVIDER" =~ vgpu.* ]]; then
        cat <<EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
  - containerPath: /dev/vfio/
    hostPath: /dev/vfio/
EOF
  fi
}

function _add_extra_portmapping() {
  if [[ "$KIND_PORT_MAPPING" != "" ]]; then
    container_port=$(echo "$KIND_PORT_MAPPING" | awk -F: '{print $1}')
    host_port=$(echo "$KIND_PORT_MAPPING" | awk -F: '{print $2}')
    if [[ -z "$container_port" || -z "$host_port" ]]; then
      echo "Invalid KIND_PORT_MAPPING format. Expected 'container_port:host_port'."
      exit 1
    fi
    cat <<EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
  extraPortMappings:
  - containerPort: $container_port
    hostPort: $host_port
EOF
  fi
}

function _add_kubeadm_cpu_manager_config_patch() {
    cat << EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
  kubeadmConfigPatches:
  - |-
    kind: JoinConfiguration
    nodeRegistration:
      kubeletExtraArgs:
        "cpu-manager-policy": "static"
        "kube-reserved": "cpu=500m"
        "system-reserved": "cpu=500m"
EOF
}

function _add_workers() {
    # appending eventual workers to the yaml
    for ((n=0;n<$(($KUBEVIRT_NUM_NODES-1));n++)); do
        cat << EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
- role: worker
EOF
    if [ $CONFIG_WORKER_CPU_MANAGER == true ]; then
         _add_kubeadm_cpu_manager_config_patch
    fi
    _add_extra_mounts
    done
}

function _add_kubeadm_config_patches_header() {
   cat <<EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
kubeadmConfigPatches:
- |
EOF
}

function _add_kubeadm_config_patches() {
  _add_kubeadm_config_patches_header
  if [ $KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY == "true" ]; then
    cat <<EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
  kind: ClusterConfiguration
  metadata:
    name: config
  etcd:
    local:
      dataDir: $ETCD_IN_MEMORY_DATA_DIR
EOF
    echo "KIND cluster etcd data will be mounted to RAM on kind nodes: $ETCD_IN_MEMORY_DATA_DIR"
  fi
  if [[ -n "$CONFIG_TOPOLOGY_MANAGER_POLICY" ]]; then
     cat <<EOF >> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
  ---
  kind: KubeletConfiguration
  topologyManagerPolicy: ${CONFIG_TOPOLOGY_MANAGER_POLICY}
  ---
EOF
  fi
}

function _setup_ipfamily() {
    if [ "$IPFAMILY" != "" ]; then
        IPFAMILY_REPLACE="networking:\n  ipFamily: $IPFAMILY"
        sed -i "s/networking:/$IPFAMILY_REPLACE/" ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
        echo "KIND cluster ip family has been set to $IPFAMILY"
    fi
}

function _prepare_kind_config() {
    _add_workers
    if [[ "$KUBEVIRT_WITH_KIND_ETCD_IN_MEMORY" == "true" ||  -n "$CONFIG_TOPOLOGY_MANAGER_POLICY" ]]; then
      _add_kubeadm_config_patches
    fi
    _setup_ipfamily
    echo "Final KIND config:"
    cat ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
}

function kind_up() {
    _fetch_kind
    _prepare_kind_config
    setup_kind
}

function _kubectl() {
    ${KUBECTL} "$@"
}

function down() {
    _fetch_kind
    if [ -z "$($KIND get clusters | grep ${CLUSTER_NAME})" ]; then
        return
    fi

    worker_nodes=$(_get_nodes | grep -i $WORKER_NODES_PATTERN | awk '{print $1}')
    for worker_node in $worker_nodes; do
        if ip netns exec $worker_node ip -details address | grep "vf 0" -B 2 > /dev/null; then
            iface=$(ip netns exec $worker_node ip -details address | grep "vf 0" -B 2 | grep -E 'UP|DOWN' | awk -F": " '{print $2}')
            ip netns exec $worker_node ip link set $iface netns 1 && echo "gracefully detached $iface from $worker_node"
        fi
    done

    # On CI, avoid failing an entire test run just because of a deletion error
    $KIND delete cluster --name=${CLUSTER_NAME} || [ "$CI" = "true" ]
    ${CRI_BIN} rm -f $REGISTRY_NAME >> /dev/null
    rm -f ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
}