vbatts/cri-o
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity
cri-o/vendor/k8s.io/kubernetes/cluster/azure-legacy/util.sh
Michał Żyłowski 52baf68d50 Vendor: Update k8s version 
Signed-off-by: Michał Żyłowski <michal.zylowski@intel.com>
2017-02-03 16:04:59 +01:00

540 lines
18 KiB
Bash
Raw Blame History

#!/bin/bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constant for the local config.
# Use the config file specified in $KUBE_CONFIG_FILE, or default to
# config-default.sh.
set -e
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/azure-legacy/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"
function prepare-e2e() {
# (e2e script runs detect-project, I don't think we need to anything)
# Note: we can't print anything here, or else the test tools will break with the extra output
return
}
function azure_call {
local -a params=()
local param
# the '... in "$@"' is implicit on a for, so doesn't need to be stated.
for param; do
params+=("${param}")
done
local rc=0
local stderr
local count=0
while [[ count -lt 10 ]]; do
stderr=$(azure "${params[@]}" 2>&1 >&3) && break
rc=$?
if [[ "${stderr}" != *"getaddrinfo ENOTFOUND"* ]]; then
break
fi
count=$(($count + 1))
done 3>&1
if [[ "${rc}" -ne 0 ]]; then
echo "${stderr}" >&2
return "${rc}"
fi
}
function json_val () {
python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1'';
}
# Verify prereqs
function verify-prereqs {
if [[ -z "$(which azure)" ]]; then
echo "Couldn't find azure in PATH"
echo " please install with 'npm install azure-cli'"
exit 1
fi
if [[ -z "$(azure_call account list | grep true)" ]]; then
echo "Default azure account not set"
echo " please set with 'azure account set'"
exit 1
fi
account=$(azure_call account list | grep true)
if which md5 > /dev/null 2>&1; then
AZ_HSH=$(md5 -q -s "$account")
else
AZ_HSH=$(echo -n "$account" | md5sum)
fi
AZ_HSH=${AZ_HSH:0:7}
AZ_STG=kube$AZ_HSH
echo "==> AZ_STG: $AZ_STG"
AZ_CS="$AZ_CS_PREFIX-$AZ_HSH"
echo "==> AZ_CS: $AZ_CS"
CONTAINER=kube-$TAG
echo "==> CONTAINER: $CONTAINER"
}
# Take the local tar files and upload them to Azure Storage. They will then be
# downloaded by the master as part of the start up script for the master.
#
# Assumed vars:
# SERVER_BINARY_TAR
# SALT_TAR
# Vars set:
# SERVER_BINARY_TAR_URL
# SALT_TAR_URL
function upload-server-tars() {
SERVER_BINARY_TAR_URL=
SALT_TAR_URL=
echo "==> SERVER_BINARY_TAR: $SERVER_BINARY_TAR"
echo "==> SALT_TAR: $SALT_TAR"
echo "+++ Staging server tars to Azure Storage: $AZ_STG"
local server_binary_url="${SERVER_BINARY_TAR##*/}"
local salt_url="${SALT_TAR##*/}"
SERVER_BINARY_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$server_binary_url"
SALT_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$salt_url"
echo "==> SERVER_BINARY_TAR_URL: $SERVER_BINARY_TAR_URL"
echo "==> SALT_TAR_URL: $SALT_TAR_URL"
echo "--> Checking storage exists..."
if [[ -z "$(azure_call storage account show $AZ_STG 2>/dev/null | \
grep data)" ]]; then
echo "--> Creating storage..."
azure_call storage account create -l "$AZ_LOCATION" $AZ_STG --type LRS
fi
echo "--> Getting storage key..."
stg_key=$(azure_call storage account keys list $AZ_STG --json | \
json_val '["primaryKey"]')
echo "--> Checking storage container exists..."
if [[ -z "$(azure_call storage container show -a $AZ_STG -k "$stg_key" \
$CONTAINER 2>/dev/null | grep data)" ]]; then
echo "--> Creating storage container..."
azure_call storage container create \
-a $AZ_STG \
-k "$stg_key" \
-p Blob \
$CONTAINER
fi
echo "--> Checking server binary exists in the container..."
if [[ -n "$(azure_call storage blob show -a $AZ_STG -k "$stg_key" \
$CONTAINER $server_binary_url 2>/dev/null | grep data)" ]]; then
echo "--> Deleting server binary in the container..."
azure_call storage blob delete \
-a $AZ_STG \
-k "$stg_key" \
$CONTAINER \
$server_binary_url
fi
echo "--> Uploading server binary to the container..."
azure_call storage blob upload \
-a $AZ_STG \
-k "$stg_key" \
$SERVER_BINARY_TAR \
$CONTAINER \
$server_binary_url
echo "--> Checking salt data exists in the container..."
if [[ -n "$(azure_call storage blob show -a $AZ_STG -k "$stg_key" \
$CONTAINER $salt_url 2>/dev/null | grep data)" ]]; then
echo "--> Deleting salt data in the container..."
azure_call storage blob delete \
-a $AZ_STG \
-k "$stg_key" \
$CONTAINER \
$salt_url
fi
echo "--> Uploading salt data to the container..."
azure_call storage blob upload \
-a $AZ_STG \
-k "$stg_key" \
$SALT_TAR \
$CONTAINER \
$salt_url
}
# Detect the information about the minions
#
# Assumed vars:
# MINION_NAMES
# ZONE
# Vars set:
#
function detect-minions () {
if [[ -z "$AZ_CS" ]]; then
verify-prereqs-local
fi
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
MINION_NAMES[$i]=$(ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net hostname -f)
done
}
# Detect the IP for the master
#
# Assumed vars:
# MASTER_NAME
# ZONE
# Vars set:
# KUBE_MASTER
# KUBE_MASTER_IP
function detect-master () {
if [[ -z "$AZ_CS" ]]; then
verify-prereqs-local
fi
KUBE_MASTER=${MASTER_NAME}
KUBE_MASTER_IP="${AZ_CS}.cloudapp.net"
echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
}
# Instantiate a kubernetes cluster
#
# Assumed vars
# KUBE_ROOT
# <Various vars set in config file>
function kube-up {
# Make sure we have the tar files staged on Azure Storage
find-release-tars
upload-server-tars
ensure-temp-dir
gen-kube-basicauth
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
local htpasswd
htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
# Generate openvpn certs
echo "--> Generating openvpn certs"
echo 01 > ${KUBE_TEMP}/ca.srl
openssl genrsa -out ${KUBE_TEMP}/ca.key
openssl req -new -x509 -days 1095 \
-key ${KUBE_TEMP}/ca.key \
-out ${KUBE_TEMP}/ca.crt \
-subj "/CN=openvpn-ca"
openssl genrsa -out ${KUBE_TEMP}/server.key
openssl req -new \
-key ${KUBE_TEMP}/server.key \
-out ${KUBE_TEMP}/server.csr \
-subj "/CN=server"
openssl x509 -req -days 1095 \
-in ${KUBE_TEMP}/server.csr \
-CA ${KUBE_TEMP}/ca.crt \
-CAkey ${KUBE_TEMP}/ca.key \
-CAserial ${KUBE_TEMP}/ca.srl \
-out ${KUBE_TEMP}/server.crt
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
openssl genrsa -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.key
openssl req -new \
-key ${KUBE_TEMP}/${MINION_NAMES[$i]}.key \
-out ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \
-subj "/CN=${MINION_NAMES[$i]}"
openssl x509 -req -days 1095 \
-in ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \
-CA ${KUBE_TEMP}/ca.crt \
-CAkey ${KUBE_TEMP}/ca.key \
-CAserial ${KUBE_TEMP}/ca.srl \
-out ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt
done
KUBE_MASTER_IP="${AZ_CS}.cloudapp.net"
# Build up start up script for master
echo "--> Building up start up script for master"
(
echo "#!/bin/bash"
echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
echo "SERVER_CRT=\"$(cat ${KUBE_TEMP}/server.crt)\""
echo "SERVER_KEY=\"$(cat ${KUBE_TEMP}/server.key)\""
echo "mkdir -p /var/cache/kubernetes-install"
echo "cd /var/cache/kubernetes-install"
echo "readonly MASTER_NAME='${MASTER_NAME}'"
echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'"
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
echo "readonly MASTER_HTPASSWD='${htpasswd}'"
echo "readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'"
echo "readonly ADMISSION_CONTROL='${ADMISSION_CONTROL:-}'"
echo "readonly KUBE_USER='${KUBE_USER}'"
echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD}'"
echo "readonly KUBE_MASTER_IP='${KUBE_MASTER_IP}'"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/common.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/create-dynamic-salt-files.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/create-kubeconfig.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/download-release.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/salt-master.sh"
) > "${KUBE_TEMP}/master-start.sh"
if [[ ! -f $AZ_SSH_KEY ]]; then
ssh-keygen -f $AZ_SSH_KEY -N ''
fi
if [[ ! -f $AZ_SSH_CERT ]]; then
openssl req -new -x509 -days 1095 -key $AZ_SSH_KEY -out $AZ_SSH_CERT \
-subj "/CN=azure-ssh-key"
fi
if [[ -z "$(azure_call network vnet show "$AZ_VNET" 2>/dev/null | grep data)" ]]; then
echo error create vnet $AZ_VNET with subnet $AZ_SUBNET
exit 1
fi
echo "--> Starting VM"
azure_call vm create \
-z "$MASTER_SIZE" \
-w "$AZ_VNET" \
-n $MASTER_NAME \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e 22000 -P \
-d ${KUBE_TEMP}/master-start.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
ssh_ports=($(eval echo "2200{1..$NUM_MINIONS}"))
#Build up start up script for minions
echo "--> Building up start up script for minions"
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
(
echo "#!/bin/bash"
echo "MASTER_NAME='${MASTER_NAME}'"
echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\""
echo "CLIENT_CRT=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt)\""
echo "CLIENT_KEY=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.key)\""
echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
echo "readonly KUBE_USER='${KUBE_USER}'"
echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD}'"
echo "readonly KUBE_MASTER_IP='${KUBE_MASTER_IP}'"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/common.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/create-kubeconfig.sh"
grep -v "^#" "${KUBE_ROOT}/cluster/azure-legacy/templates/salt-minion.sh"
) > "${KUBE_TEMP}/minion-start-${i}.sh"
echo "--> Starting VM"
azure_call vm create \
-z "$MINION_SIZE" \
-c -w "$AZ_VNET" \
-n ${MINION_NAMES[$i]} \
-l "$AZ_LOCATION" \
-t $AZ_SSH_CERT \
-e ${ssh_ports[$i]} -P \
-d ${KUBE_TEMP}/minion-start-${i}.sh \
-b $AZ_SUBNET \
$AZ_CS $AZ_IMAGE $USER
done
echo "--> Creating endpoint"
azure_call vm endpoint create $MASTER_NAME 443
detect-master > /dev/null
echo "==> KUBE_MASTER_IP: ${KUBE_MASTER_IP}"
echo "Waiting for cluster initialization."
echo
echo " This will continually check to see if the API for kubernetes is reachable."
echo " This might loop forever if there was some uncaught error during start"
echo " up."
echo
until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
--fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/healthz"; do
printf "."
sleep 2
done
printf "\n"
echo "Kubernetes cluster created."
export CONTEXT="azure_${INSTANCE_PREFIX}"
create-kubeconfig
export KUBE_CERT="/tmp/$RANDOM-kubecfg.crt"
export KUBE_KEY="/tmp/$RANDOM-kubecfg.key"
export CA_CERT="/tmp/$RANDOM-kubernetes.ca.crt"
# TODO: generate ADMIN (and KUBELET) tokens and put those in the master's
# config file. Distribute the same way the htpasswd is done.
(umask 077
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cat /srv/kubernetes/kubecfg.crt >"${KUBE_CERT}" 2>/dev/null
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cat /srv/kubernetes/kubecfg.key >"${KUBE_KEY}" 2>/dev/null
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cat /srv/kubernetes/ca.crt >"${CA_CERT}" 2>/dev/null
)
echo "Sanity checking cluster..."
echo
echo " This will continually check the minions to ensure docker is"
echo " installed. This is usually a good indicator that salt has"
echo " successfully provisioned. This might loop forever if there was"
echo " some uncaught error during start up."
echo
# Basic sanity checking
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Make sure docker is installed
echo "--> Making sure docker is installed on ${MINION_NAMES[$i]}."
until ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} \
$AZ_CS.cloudapp.net which docker > /dev/null 2>&1; do
printf "."
sleep 2
done
done
sleep 60
KUBECONFIG_NAME="kubeconfig"
KUBECONFIG="${HOME}/.kube/config"
echo "Distributing kubeconfig for kubelet to master kubelet"
scp -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -P 22000 ${KUBECONFIG} \
$AZ_CS.cloudapp.net:${KUBECONFIG_NAME}
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo cp ${KUBECONFIG_NAME} /var/lib/kubelet/${KUBECONFIG_NAME}
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p 22000 $AZ_CS.cloudapp.net \
sudo service kubelet restart
echo "Distributing kubeconfig for kubelet to all minions"
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
scp -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -P ${ssh_ports[$i]} ${KUBECONFIG} \
$AZ_CS.cloudapp.net:${KUBECONFIG_NAME}
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net \
sudo cp ${KUBECONFIG_NAME} /var/lib/kubelet/${KUBECONFIG_NAME}
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net \
sudo cp ${KUBECONFIG_NAME} /var/lib/kube-proxy/${KUBECONFIG_NAME}
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net \
sudo service kubelet restart
ssh -oStrictHostKeyChecking=no -i $AZ_SSH_KEY -p ${ssh_ports[$i]} $AZ_CS.cloudapp.net \
sudo killall kube-proxy
done
# ensures KUBECONFIG is set
get-kubeconfig-basicauth
echo
echo "Kubernetes cluster is running. The master is running at:"
echo
echo " https://${KUBE_MASTER_IP}"
echo
echo "The user name and password to use is located in ${KUBECONFIG}."
echo
}
# Delete a kubernetes cluster
function kube-down {
echo "Bringing down cluster"
set +e
azure_call vm delete $MASTER_NAME -b -q
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
azure_call vm delete ${MINION_NAMES[$i]} -b -q
done
wait
}
# Update a kubernetes cluster with latest source
#function kube-push {
# detect-project
# detect-master
# Make sure we have the tar files staged on Azure Storage
# find-release-tars
# upload-server-tars
# (
# echo "#! /bin/bash"
# echo "mkdir -p /var/cache/kubernetes-install"
# echo "cd /var/cache/kubernetes-install"
# echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
# echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
# grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/common.sh"
# grep -v "^#" "${KUBE_ROOT}/cluster/azure/templates/download-release.sh"
# echo "echo Executing configuration"
# echo "sudo salt '*' mine.update"
# echo "sudo salt --force-color '*' state.highstate"
# ) | gcutil ssh --project "$PROJECT" --zone "$ZONE" "$KUBE_MASTER" sudo bash
# get-kubeconfig-basicauth
# echo
# echo "Kubernetes cluster is running. The master is running at:"
# echo
# echo " https://${KUBE_MASTER_IP}"
# echo
# echo "The user name and password to use is located in ${KUBECONFIG:-$DEFAULT_KUBECONFIG}."
# echo
#}
# -----------------------------------------------------------------------------
# Cluster specific test helpers
# Execute prior to running tests to build a release if required for env.
#
# Assumed Vars:
# KUBE_ROOT
function test-build-release {
# Make a release
"${KUBE_ROOT}/build/release.sh"
}
# SSH to a node by name ($1) and run a command ($2).
function ssh-to-node {
local node="$1"
local cmd="$2"
ssh --ssh_arg "-o LogLevel=quiet" "${node}" "${cmd}"
}
# Restart the kube-proxy on a node ($1)
function restart-kube-proxy {
ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart"
}
# Restart the kube-proxy on the master ($1)
function restart-apiserver {
ssh-to-node "$1" "sudo /etc/init.d/kube-apiserver restart"
}
function test-setup {
"${KUBE_ROOT}/cluster/kube-up.sh"
}
function test-teardown {
"${KUBE_ROOT}/cluster/kube-down.sh"
}
Reference in a new issue View git blame Copy permalink