cluster/aws/util.sh

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A library of helper functions and constant for the local config.

# Experimental flags can be removed/renamed at any time.
# The intent is to allow experimentation/advanced functionality before we
# are ready to commit to supporting it.
# Experimental functionality:
#   KUBE_USE_EXISTING_MASTER=true
#     Detect and reuse an existing master; useful if you want to
#     create more nodes, perhaps with a different instance type or in
#     a different subnet/AZ
#   KUBE_SUBNET_CIDR=172.20.1.0/24
#     Override the default subnet CIDR; useful if you want to create
#     a second subnet.  The default subnet is 172.20.0.0/24.  The VPC
#     is created with 172.20.0.0/16; you must pick a sub-CIDR of that.

# Use the config file specified in $KUBE_CONFIG_FILE, or default to
# config-default.sh.
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/aws/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"
source "${KUBE_ROOT}/cluster/lib/util.sh"

ALLOCATE_NODE_CIDRS=true

NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion"

# The Auto Scaling Group (ASG) name must be unique, so we include the zone
ASG_NAME="${NODE_INSTANCE_PREFIX}-group-${ZONE}"

# We could allow the master disk volume id to be specified in future
MASTER_DISK_ID=

# Well known tags
TAG_KEY_MASTER_IP="kubernetes.io/master-ip"

OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION}

# Defaults: ubuntu -> wily
if [[ "${OS_DISTRIBUTION}" == "ubuntu" ]]; then
  OS_DISTRIBUTION=wily
fi

# Loads the distro-specific utils script.
# If the distro is not recommended, prints warnings or exits.
function load_distro_utils () {
case "${OS_DISTRIBUTION}" in
  jessie)
    ;;
  wily)
    ;;
  vivid)
    echo "vivid is no longer supported by kube-up; please use jessie instead" >&2
    exit 2
    ;;
  coreos)
    echo "coreos is no longer supported by kube-up; please use jessie instead" >&2
    exit 2
    ;;
  trusty)
    echo "trusty is no longer supported by kube-up; please use jessie or wily instead" >&2
    exit 2
    ;;
  wheezy)
    echo "wheezy is no longer supported by kube-up; please use jessie instead" >&2
    exit 2
    ;;
  *)
    echo "Cannot start cluster using os distro: ${OS_DISTRIBUTION}" >&2
    echo "The current recommended distro is jessie" >&2
    exit 2
    ;;
esac

source "${KUBE_ROOT}/cluster/aws/${OS_DISTRIBUTION}/util.sh"
}

load_distro_utils

# This removes the final character in bash (somehow)
re='[a-zA-Z]'
if [[ ${ZONE: -1} =~ $re  ]]; then 
  AWS_REGION=${ZONE%?}
else 
  AWS_REGION=$ZONE
fi

export AWS_DEFAULT_REGION=${AWS_REGION}
export AWS_DEFAULT_OUTPUT=text
AWS_CMD="aws ec2"
AWS_ASG_CMD="aws autoscaling"

VPC_CIDR_BASE=${KUBE_VPC_CIDR_BASE:-172.20}
MASTER_IP_SUFFIX=.9
VPC_CIDR=${VPC_CIDR_BASE}.0.0/16
SUBNET_CIDR=${VPC_CIDR_BASE}.0.0/24
if [[ -n "${KUBE_SUBNET_CIDR:-}" ]]; then
  echo "Using subnet CIDR override: ${KUBE_SUBNET_CIDR}"
  SUBNET_CIDR=${KUBE_SUBNET_CIDR}
fi
if [[ -z "${MASTER_INTERNAL_IP-}" ]]; then
  MASTER_INTERNAL_IP="${SUBNET_CIDR%.*}${MASTER_IP_SUFFIX}"
fi

MASTER_SG_NAME="kubernetes-master-${CLUSTER_ID}"
NODE_SG_NAME="kubernetes-minion-${CLUSTER_ID}"

IAM_PROFILE_MASTER="kubernetes-master-${CLUSTER_ID}-${VPC_NAME}"
IAM_PROFILE_NODE="kubernetes-minion-${CLUSTER_ID}-${VPC_NAME}"

# Be sure to map all the ephemeral drives.  We can specify more than we actually have.
# TODO: Actually mount the correct number (especially if we have more), though this is non-trivial, and
#  only affects the big storage instance types, which aren't a typical use case right now.
EPHEMERAL_BLOCK_DEVICE_MAPPINGS=",{\"DeviceName\": \"/dev/sdc\",\"VirtualName\":\"ephemeral0\"},{\"DeviceName\": \"/dev/sdd\",\"VirtualName\":\"ephemeral1\"},{\"DeviceName\": \"/dev/sde\",\"VirtualName\":\"ephemeral2\"},{\"DeviceName\": \"/dev/sdf\",\"VirtualName\":\"ephemeral3\"}"

# Experimental: If the user sets KUBE_AWS_STORAGE to ebs, use ebs storage
# in preference to local instance storage We do this by not mounting any
# instance storage.  We could do this better in future (e.g. making instance
# storage available for other purposes)
if [[ "${KUBE_AWS_STORAGE:-}" == "ebs" ]]; then
  EPHEMERAL_BLOCK_DEVICE_MAPPINGS=""
fi

# TODO (bburns) Parameterize this for multiple cluster per project
function get_vpc_id {
  $AWS_CMD describe-vpcs \
           --filters Name=tag:Name,Values=${VPC_NAME} \
                     Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
           --query Vpcs[].VpcId
}

function get_subnet_id {
  local vpc_id=$1
  local az=$2
  $AWS_CMD describe-subnets \
           --filters Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                     Name=availabilityZone,Values=${az} \
                     Name=vpc-id,Values=${vpc_id} \
           --query Subnets[].SubnetId
}

function get_igw_id {
  local vpc_id=$1
  $AWS_CMD describe-internet-gateways \
           --filters Name=attachment.vpc-id,Values=${vpc_id} \
           --query InternetGateways[].InternetGatewayId
}

function get_elbs_in_vpc {
  # ELB doesn't seem to be on the same platform as the rest of AWS; doesn't support filtering
  aws elb --output json describe-load-balancers  | \
    python -c "import json,sys; lst = [str(lb['LoadBalancerName']) for lb in json.load(sys.stdin)['LoadBalancerDescriptions'] if 'VPCId' in lb and lb['VPCId'] == '$1']; print('\n'.join(lst))"
}

function get_instanceid_from_name {
  local tagName=$1
  $AWS_CMD describe-instances \
    --filters Name=tag:Name,Values=${tagName} \
              Name=instance-state-name,Values=running \
              Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
    --query Reservations[].Instances[].InstanceId
}

function get_instance_public_ip {
  local instance_id=$1
  $AWS_CMD describe-instances \
    --instance-ids ${instance_id} \
    --query Reservations[].Instances[].NetworkInterfaces[0].Association.PublicIp
}

function get_instance_private_ip {
  local instance_id=$1
  $AWS_CMD describe-instances \
    --instance-ids ${instance_id} \
    --query Reservations[].Instances[].NetworkInterfaces[0].PrivateIpAddress
}

# Gets a security group id, by name ($1)
function get_security_group_id {
  local name=$1
  $AWS_CMD describe-security-groups \
           --filters Name=vpc-id,Values=${VPC_ID} \
                     Name=group-name,Values=${name} \
                     Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
           --query SecurityGroups[].GroupId \
  | tr "\t" "\n"
}

# Finds the master ip, if it is saved (tagged on the master disk)
# Sets KUBE_MASTER_IP
function find-tagged-master-ip {
  find-master-pd
  if [[ -n "${MASTER_DISK_ID:-}" ]]; then
    KUBE_MASTER_IP=$(get-tag ${MASTER_DISK_ID} ${TAG_KEY_MASTER_IP})
  fi
}

# Gets a tag value from an AWS resource
# usage: get-tag <resource-id> <tag-name>
# outputs: the tag value, or "" if no tag
function get-tag {
  $AWS_CMD describe-tags --filters Name=resource-id,Values=${1} \
                                   Name=key,Values=${2} \
                         --query Tags[].Value
}

# Gets an existing master, exiting if not found
# Note that this is called directly by the e2e tests
function detect-master() {
  find-tagged-master-ip
  KUBE_MASTER=${MASTER_NAME}
  if [[ -z "${KUBE_MASTER_IP:-}" ]]; then
    echo "Could not detect Kubernetes master node IP.  Make sure you've launched a cluster with 'kube-up.sh'"
    exit 1
  fi
  echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
}

# Reads kube-env metadata from master
#
# Assumed vars:
#   KUBE_MASTER_IP
#   AWS_SSH_KEY
#   SSH_USER
function get-master-env() {
  ssh -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${KUBE_MASTER_IP} sudo cat /etc/kubernetes/kube_env.yaml
}


function query-running-minions () {
  local query=$1
  $AWS_CMD describe-instances \
           --filters Name=instance-state-name,Values=running \
                     Name=vpc-id,Values=${VPC_ID} \
                     Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                     Name=tag:aws:autoscaling:groupName,Values=${ASG_NAME} \
                     Name=tag:Role,Values=${NODE_TAG} \
           --query ${query}
}

function detect-node-names () {
  # If this is called directly, VPC_ID might not be set
  # (this is case from cluster/log-dump.sh)
  if [[ -z "${VPC_ID:-}" ]]; then
    VPC_ID=$(get_vpc_id)
  fi

  NODE_IDS=()
  NODE_NAMES=()
  for id in $(query-running-minions "Reservations[].Instances[].InstanceId"); do
    NODE_IDS+=("${id}")

    # We use the minion ids as the name
    NODE_NAMES+=("${id}")
  done
}

# Called to detect the project on GCE
# Not needed on AWS
function detect-project() {
  :
}

function detect-nodes () {
  detect-node-names

  # This is inefficient, but we want NODE_NAMES / NODE_IDS to be ordered the same as KUBE_NODE_IP_ADDRESSES
  KUBE_NODE_IP_ADDRESSES=()
  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
    local minion_ip
    if [[ "${ENABLE_NODE_PUBLIC_IP}" == "true" ]]; then
      minion_ip=$(get_instance_public_ip ${NODE_NAMES[$i]})
    else
      minion_ip=$(get_instance_private_ip ${NODE_NAMES[$i]})
    fi
    echo "Found minion ${i}: ${NODE_NAMES[$i]} @ ${minion_ip}"
    KUBE_NODE_IP_ADDRESSES+=("${minion_ip}")
  done

  if [[ -z "$KUBE_NODE_IP_ADDRESSES" ]]; then
    echo "Could not detect Kubernetes minion nodes.  Make sure you've launched a cluster with 'kube-up.sh'"
    exit 1
  fi
}

function detect-security-groups {
  if [[ -z "${MASTER_SG_ID-}" ]]; then
    MASTER_SG_ID=$(get_security_group_id "${MASTER_SG_NAME}")
    if [[ -z "${MASTER_SG_ID}" ]]; then
      echo "Could not detect Kubernetes master security group.  Make sure you've launched a cluster with 'kube-up.sh'"
      exit 1
    else
      echo "Using master security group: ${MASTER_SG_NAME} ${MASTER_SG_ID}"
    fi
  fi
  if [[ -z "${NODE_SG_ID-}" ]]; then
    NODE_SG_ID=$(get_security_group_id "${NODE_SG_NAME}")
    if [[ -z "${NODE_SG_ID}" ]]; then
      echo "Could not detect Kubernetes minion security group.  Make sure you've launched a cluster with 'kube-up.sh'"
      exit 1
    else
      echo "Using minion security group: ${NODE_SG_NAME} ${NODE_SG_ID}"
    fi
  fi
}

# Detects the AMI to use (considering the region)
# This really should be in the various distro-specific util functions,
# but CoreOS uses this for the master, so for now it is here.
#
# TODO: Remove this and just have each distro implement detect-image
#
# Vars set:
#   AWS_IMAGE
function detect-image () {
case "${OS_DISTRIBUTION}" in
  wily)
    detect-wily-image
    ;;
  jessie)
    detect-jessie-image
    ;;
  *)
    echo "Please specify AWS_IMAGE directly (distro ${OS_DISTRIBUTION} not recognized)"
    exit 2
    ;;
esac
}

# Detects the RootDevice to use in the Block Device Mapping (considering the AMI)
#
# Vars set:
#   MASTER_BLOCK_DEVICE_MAPPINGS
#   NODE_BLOCK_DEVICE_MAPPINGS
#
function detect-root-device {
  local master_image=${AWS_IMAGE}
  local node_image=${KUBE_NODE_IMAGE}

  ROOT_DEVICE_MASTER=$($AWS_CMD describe-images --image-ids ${master_image} --query 'Images[].RootDeviceName')
  if [[ "${master_image}" == "${node_image}" ]]; then
      ROOT_DEVICE_NODE=${ROOT_DEVICE_MASTER}
    else
      ROOT_DEVICE_NODE=$($AWS_CMD describe-images --image-ids ${node_image} --query 'Images[].RootDeviceName')
  fi

  MASTER_BLOCK_DEVICE_MAPPINGS="[{\"DeviceName\":\"${ROOT_DEVICE_MASTER}\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":${MASTER_ROOT_DISK_SIZE},\"VolumeType\":\"${MASTER_ROOT_DISK_TYPE}\"}} ${EPHEMERAL_BLOCK_DEVICE_MAPPINGS}]"
  NODE_BLOCK_DEVICE_MAPPINGS="[{\"DeviceName\":\"${ROOT_DEVICE_NODE}\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":${NODE_ROOT_DISK_SIZE},\"VolumeType\":\"${NODE_ROOT_DISK_TYPE}\"}} ${EPHEMERAL_BLOCK_DEVICE_MAPPINGS}]"
}

# Computes the AWS fingerprint for a public key file ($1)
# $1: path to public key file
# Note that this is a different hash from the OpenSSH hash.
# But AWS gives us this public key hash in the describe keys output, so we should stick with this format.
# Hopefully this will be done by the aws cli tool one day: https://github.com/aws/aws-cli/issues/191
# NOTE: This does not work on Mavericks, due to an odd ssh-keygen version, so we use get-ssh-fingerprint instead
function get-aws-fingerprint {
  local -r pubkey_path=$1
  ssh-keygen -f ${pubkey_path} -e -m PKCS8  | openssl rsa -pubin -outform DER | openssl md5 -c | sed -e 's/(stdin)= //g'
}

# Computes the SSH fingerprint for a public key file ($1)
# #1: path to public key file
# Note this is different from the AWS fingerprint; see notes on get-aws-fingerprint
function get-ssh-fingerprint {
  local -r pubkey_path=$1
  ssh-keygen -lf ${pubkey_path} | cut -f2 -d' '
}

# Import an SSH public key to AWS.
# Ignores duplicate names; recommended to use a name that includes the public key hash.
# $1 name
# $2 public key path
function import-public-key {
  local -r name=$1
  local -r path=$2

  local ok=1
  local output=""
  output=$($AWS_CMD import-key-pair --key-name ${name} --public-key-material "file://${path}" 2>&1) || ok=0
  if [[ ${ok} == 0 ]]; then
    # Idempotency: ignore if duplicate name
    if [[ "${output}" != *"InvalidKeyPair.Duplicate"* ]]; then
      echo "Error importing public key"
      echo "Output: ${output}"
      exit 1
    fi
  fi
}

# Robustly try to create a security group, if it does not exist.
# $1: The name of security group; will be created if not exists
# $2: Description for security group (used if created)
#
# Note that this doesn't actually return the sgid; we need to re-query
function create-security-group {
  local -r name=$1
  local -r description=$2

  local sgid=$(get_security_group_id "${name}")
  if [[ -z "$sgid" ]]; then
	  echo "Creating security group ${name}."
	  sgid=$($AWS_CMD create-security-group --group-name "${name}" --description "${description}" --vpc-id "${VPC_ID}" --query GroupId)
	  add-tag $sgid KubernetesCluster ${CLUSTER_ID}
  fi
}

# Authorize ingress to a security group.
# Attempts to be idempotent, though we end up checking the output looking for error-strings.
# $1 group-id
# $2.. arguments to pass to authorize-security-group-ingress
function authorize-security-group-ingress {
  local -r sgid=$1
  shift
  local ok=1
  local output=""
  output=$($AWS_CMD authorize-security-group-ingress --group-id "${sgid}" $@ 2>&1) || ok=0
  if [[ ${ok} == 0 ]]; then
    # Idempotency: ignore if duplicate rule
    if [[ "${output}" != *"InvalidPermission.Duplicate"* ]]; then
      echo "Error creating security group ingress rule"
      echo "Output: ${output}"
      exit 1
    fi
  fi
}

# Gets master persistent volume, if exists
# Sets MASTER_DISK_ID
function find-master-pd {
  local name=${MASTER_NAME}-pd
  if [[ -z "${MASTER_DISK_ID}" ]]; then
    local zone_filter="Name=availability-zone,Values=${ZONE}"
    if [[ "${KUBE_USE_EXISTING_MASTER:-}" == "true" ]]; then
      # If we're reusing an existing master, it is likely to be in another zone
      # If running multizone, your cluster must be uniquely named across zones
      zone_filter=""
    fi
    MASTER_DISK_ID=`$AWS_CMD describe-volumes \
                             --filters ${zone_filter} \
                                       Name=tag:Name,Values=${name} \
                                       Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                             --query Volumes[].VolumeId`
  fi
}

# Gets or creates master persistent volume
# Sets MASTER_DISK_ID
function ensure-master-pd {
  local name=${MASTER_NAME}-pd

  find-master-pd

  if [[ -z "${MASTER_DISK_ID}" ]]; then
    echo "Creating master disk: size ${MASTER_DISK_SIZE}GB, type ${MASTER_DISK_TYPE}"
    MASTER_DISK_ID=`$AWS_CMD create-volume --availability-zone ${ZONE} --volume-type ${MASTER_DISK_TYPE} --size ${MASTER_DISK_SIZE} --query VolumeId`
    add-tag ${MASTER_DISK_ID} Name ${name}
    add-tag ${MASTER_DISK_ID} KubernetesCluster ${CLUSTER_ID}
  fi
}

# Configures a CloudWatch alarm to reboot the instance on failure
function reboot-on-failure {
  local instance_id=$1

  echo "Creating Cloudwatch alarm to reboot instance ${instance_id} on failure"

  local aws_owner_id=`aws ec2 describe-instances --instance-ids ${instance_id} --query Reservations[0].OwnerId`
  if [[ -z "${aws_owner_id}" ]]; then
    echo "Unable to determinate AWS account id for ${instance_id}"
    exit 1
  fi

  aws cloudwatch put-metric-alarm \
                 --alarm-name k8s-${instance_id}-statuscheckfailure-reboot \
                 --alarm-description "Reboot ${instance_id} on status check failure" \
                 --namespace "AWS/EC2" \
                 --dimensions Name=InstanceId,Value=${instance_id} \
                 --statistic Minimum \
                 --metric-name StatusCheckFailed \
                 --comparison-operator GreaterThanThreshold \
                 --threshold 0 \
                 --period 60 \
                 --evaluation-periods 3 \
                 --alarm-actions arn:aws:swf:${AWS_REGION}:${aws_owner_id}:action/actions/AWS_EC2.InstanceId.Reboot/1.0 > $LOG

  # TODO: The IAM role EC2ActionsAccess must have been created
  # See e.g. http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/UsingIAM.html
}

function delete-instance-alarms {
  local instance_id=$1

  alarm_names=`aws cloudwatch describe-alarms --alarm-name-prefix k8s-${instance_id}- --query MetricAlarms[].AlarmName`
  for alarm_name in ${alarm_names}; do
    aws cloudwatch delete-alarms --alarm-names ${alarm_name} > $LOG
  done
}

# Finds the existing master IP, or creates/reuses an Elastic IP
# If MASTER_RESERVED_IP looks like an IP address, we will use it;
# otherwise we will create a new elastic IP
# Sets KUBE_MASTER_IP
function ensure-master-ip {
  find-tagged-master-ip

  if [[ -z "${KUBE_MASTER_IP:-}" ]]; then
    # Check if MASTER_RESERVED_IP looks like an IPv4 address
    # Note that we used to only allocate an elastic IP when MASTER_RESERVED_IP=auto
    # So be careful changing the IPV4 test, to be sure that 'auto' => 'allocate'
    if [[ "${MASTER_RESERVED_IP}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
      KUBE_MASTER_IP="${MASTER_RESERVED_IP}"
    else
      KUBE_MASTER_IP=`$AWS_CMD allocate-address --domain vpc --query PublicIp`
      echo "Allocated Elastic IP for master: ${KUBE_MASTER_IP}"
    fi

    # We can't tag elastic ips.  Instead we put the tag on the persistent disk.
    # It is a little weird, perhaps, but it sort of makes sense...
    # The master mounts the master PD, and whoever mounts the master PD should also
    # have the master IP
    add-tag ${MASTER_DISK_ID} ${TAG_KEY_MASTER_IP} ${KUBE_MASTER_IP}
  fi
}

# Creates a new DHCP option set configured correctly for Kubernetes when DHCP_OPTION_SET_ID is not specified
# Sets DHCP_OPTION_SET_ID
function create-dhcp-option-set () {
  if [[ -z ${DHCP_OPTION_SET_ID-} ]]; then
    case "${AWS_REGION}" in
      us-east-1)
        OPTION_SET_DOMAIN=ec2.internal
        ;;

      *)
        OPTION_SET_DOMAIN="${AWS_REGION}.compute.internal"
    esac

    DHCP_OPTION_SET_ID=$($AWS_CMD create-dhcp-options --dhcp-configuration Key=domain-name,Values=${OPTION_SET_DOMAIN} Key=domain-name-servers,Values=AmazonProvidedDNS --query DhcpOptions.DhcpOptionsId)

    add-tag ${DHCP_OPTION_SET_ID} Name kubernetes-dhcp-option-set
    add-tag ${DHCP_OPTION_SET_ID} KubernetesCluster ${CLUSTER_ID}
  fi

  $AWS_CMD associate-dhcp-options --dhcp-options-id ${DHCP_OPTION_SET_ID} --vpc-id ${VPC_ID} > $LOG

  echo "Using DHCP option set ${DHCP_OPTION_SET_ID}"
}

# Verify prereqs
function verify-prereqs {
  if [[ "$(which aws)" == "" ]]; then
    echo "Can't find aws in PATH, please fix and retry."
    exit 1
  fi
}


# Create a temp dir that'll be deleted at the end of this bash session.
#
# Vars set:
#   KUBE_TEMP
function ensure-temp-dir {
  if [[ -z ${KUBE_TEMP-} ]]; then
    KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
    trap 'rm -rf "${KUBE_TEMP}"' EXIT
  fi
}

# Take the local tar files and upload them to S3.  They will then be
# downloaded by the master as part of the start up script for the master.
#
# Assumed vars:
#   SERVER_BINARY_TAR
#   SALT_TAR
# Vars set:
#   SERVER_BINARY_TAR_URL
#   SALT_TAR_URL
function upload-server-tars() {
  SERVER_BINARY_TAR_URL=
  SERVER_BINARY_TAR_HASH=
  SALT_TAR_URL=
  SALT_TAR_HASH=
  BOOTSTRAP_SCRIPT_URL=
  BOOTSTRAP_SCRIPT_HASH=

  ensure-temp-dir

  SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
  SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
  BOOTSTRAP_SCRIPT_HASH=$(sha1sum-file "${BOOTSTRAP_SCRIPT}")

  if [[ -z ${AWS_S3_BUCKET-} ]]; then
      local project_hash=
      local key=$(aws configure get aws_access_key_id)
      if which md5 > /dev/null 2>&1; then
        project_hash=$(md5 -q -s "${USER} ${key} ${INSTANCE_PREFIX}")
      else
        project_hash=$(echo -n "${USER} ${key} ${INSTANCE_PREFIX}" | md5sum | awk '{ print $1 }')
      fi
      AWS_S3_BUCKET="kubernetes-staging-${project_hash}"
  fi

  echo "Uploading to Amazon S3"

  if ! aws s3api get-bucket-location --bucket ${AWS_S3_BUCKET} > /dev/null 2>&1 ; then
    echo "Creating ${AWS_S3_BUCKET}"

    # Buckets must be globally uniquely named, so always create in a known region
    # We default to us-east-1 because that's the canonical region for S3,
    # and then the bucket is most-simply named (s3.amazonaws.com)
    aws s3 mb "s3://${AWS_S3_BUCKET}" --region ${AWS_S3_REGION}

    echo "Confirming bucket was created..."

    local attempt=0
    while true; do
      if ! aws s3 ls --region ${AWS_S3_REGION} "s3://${AWS_S3_BUCKET}" > /dev/null 2>&1; then
        if (( attempt > 120 )); then
          echo
          echo -e "${color_red}Unable to confirm bucket creation." >&2
          echo "Please ensure that s3://${AWS_S3_BUCKET} exists" >&2
          echo -e "and run the script again. (sorry!)${color_norm}" >&2
          exit 1
        fi
      else
        break
      fi
      attempt=$(($attempt+1))
      sleep 1
    done
  fi

  local s3_bucket_location=$(aws s3api get-bucket-location --bucket ${AWS_S3_BUCKET})
  local s3_url_base=https://s3-${s3_bucket_location}.amazonaws.com
  if [[ "${s3_bucket_location}" == "None" ]]; then
    # "US Classic" does not follow the pattern
    s3_url_base=https://s3.amazonaws.com
    s3_bucket_location=us-east-1
  elif [[ "${s3_bucket_location}" == "cn-north-1" ]]; then
    s3_url_base=https://s3.cn-north-1.amazonaws.com.cn
  fi

  local -r staging_path="devel"

  local -r local_dir="${KUBE_TEMP}/s3/"
  mkdir ${local_dir}

  echo "+++ Staging server tars to S3 Storage: ${AWS_S3_BUCKET}/${staging_path}"
  cp -a "${SERVER_BINARY_TAR}" ${local_dir}
  cp -a "${SALT_TAR}" ${local_dir}
  cp -a "${BOOTSTRAP_SCRIPT}" ${local_dir}

  aws s3 sync --region ${s3_bucket_location} --exact-timestamps ${local_dir} "s3://${AWS_S3_BUCKET}/${staging_path}/"

  local server_binary_path="${staging_path}/${SERVER_BINARY_TAR##*/}"
  aws s3api put-object-acl --region ${s3_bucket_location} --bucket ${AWS_S3_BUCKET} --key "${server_binary_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
  SERVER_BINARY_TAR_URL="${s3_url_base}/${AWS_S3_BUCKET}/${server_binary_path}"

  local salt_tar_path="${staging_path}/${SALT_TAR##*/}"
  aws s3api put-object-acl --region ${s3_bucket_location} --bucket ${AWS_S3_BUCKET} --key "${salt_tar_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
  SALT_TAR_URL="${s3_url_base}/${AWS_S3_BUCKET}/${salt_tar_path}"

  local bootstrap_script_path="${staging_path}/${BOOTSTRAP_SCRIPT##*/}"
  aws s3api put-object-acl --region ${s3_bucket_location} --bucket ${AWS_S3_BUCKET} --key "${bootstrap_script_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
  BOOTSTRAP_SCRIPT_URL="${s3_url_base}/${AWS_S3_BUCKET}/${bootstrap_script_path}"

  echo "Uploaded server tars:"
  echo "  SERVER_BINARY_TAR_URL: ${SERVER_BINARY_TAR_URL}"
  echo "  SALT_TAR_URL: ${SALT_TAR_URL}"
  echo "  BOOTSTRAP_SCRIPT_URL: ${BOOTSTRAP_SCRIPT_URL}"
}

# Adds a tag to an AWS resource
# usage: add-tag <resource-id> <tag-name> <tag-value>
function add-tag {
  echo "Adding tag to ${1}: ${2}=${3}"

  # We need to retry in case the resource isn't yet fully created
  n=0
  until [ $n -ge 25 ]; do
    $AWS_CMD create-tags --resources ${1} --tags Key=${2},Value=${3} > $LOG && return
    n=$[$n+1]
    sleep 3
  done

  echo "Unable to add tag to AWS resource"
  exit 1
}

# Creates the IAM profile, based on configuration files in templates/iam
# usage: create-iam-profile kubernetes-master-us-west-1a-chom kubernetes-master
function create-iam-profile {
  local key=$1
  local role=$2

  local conf_dir=file://${KUBE_ROOT}/cluster/aws/templates/iam

  echo "Creating IAM role: ${key}"
  aws iam create-role --role-name ${key} --assume-role-policy-document ${conf_dir}/${role}-role.json > $LOG

  echo "Creating IAM role-policy: ${key}"
  aws iam put-role-policy --role-name ${key} --policy-name ${key} --policy-document ${conf_dir}/${role}-policy.json > $LOG

  echo "Creating IAM instance-policy: ${key}"
  aws iam create-instance-profile --instance-profile-name ${key} > $LOG

  echo "Adding IAM role to instance-policy: ${key}"
  aws iam add-role-to-instance-profile --instance-profile-name ${key} --role-name ${key} > $LOG
}

# Creates the IAM roles (if they do not already exist)
function ensure-iam-profiles {
  echo "Creating master IAM profile: ${IAM_PROFILE_MASTER}"
  create-iam-profile ${IAM_PROFILE_MASTER} kubernetes-master

  echo "Creating minion IAM profile: ${IAM_PROFILE_NODE}"
  create-iam-profile ${IAM_PROFILE_NODE} kubernetes-minion
}

# Wait for instance to be in specified state
function wait-for-instance-state {
  instance_id=$1
  state=$2

  while true; do
    instance_state=$($AWS_CMD describe-instances --instance-ids ${instance_id} --query Reservations[].Instances[].State.Name)
    if [[ "$instance_state" == "${state}" ]]; then
      break
    else
      echo "Waiting for instance ${instance_id} to be ${state} (currently ${instance_state})"
      echo "Sleeping for 3 seconds..."
      sleep 3
    fi
  done
}

# Allocates new Elastic IP from Amazon
# Output: allocated IP address
function allocate-elastic-ip {
  $AWS_CMD allocate-address --domain vpc --query PublicIp
}

# Attaches an elastic IP to the specified instance
function attach-ip-to-instance {
  local ip_address=$1
  local instance_id=$2

  local elastic_ip_allocation_id=$($AWS_CMD describe-addresses --public-ips $ip_address --query Addresses[].AllocationId)
  echo "Attaching IP ${ip_address} to instance ${instance_id}"
  $AWS_CMD associate-address --instance-id ${instance_id} --allocation-id ${elastic_ip_allocation_id} > $LOG
}

# Releases an elastic IP
function release-elastic-ip {
  local ip_address=$1

  echo "Releasing Elastic IP: ${ip_address}"
  elastic_ip_allocation_id=$($AWS_CMD describe-addresses --public-ips $ip_address --query Addresses[].AllocationId 2> $LOG) || true
  if [[ -z "${elastic_ip_allocation_id}" ]]; then
    echo "Elastic IP already released"
  else
    $AWS_CMD release-address --allocation-id ${elastic_ip_allocation_id} > $LOG
  fi
}

# Deletes a security group
# usage: delete_security_group <sgid>
function delete_security_group {
  local -r sg_id=${1}

  echo "Deleting security group: ${sg_id}"

  # We retry in case there's a dependent resource - typically an ELB
  local n=0
  until [ $n -ge 20 ]; do
    $AWS_CMD delete-security-group --group-id ${sg_id} > $LOG && return
    n=$[$n+1]
    sleep 3
  done
  echo "Unable to delete security group: ${sg_id}"
  exit 1
}


# Deletes master and minion IAM roles and instance profiles
# usage: delete-iam-instance-profiles
function delete-iam-profiles {
  for iam_profile_name in ${IAM_PROFILE_MASTER} ${IAM_PROFILE_NODE};do
    echo "Removing role from instance profile: ${iam_profile_name}"
    conceal-no-such-entity-response aws iam remove-role-from-instance-profile --instance-profile-name "${iam_profile_name}" --role-name "${iam_profile_name}"

    echo "Deleting IAM Instance-Profile: ${iam_profile_name}"
    conceal-no-such-entity-response aws iam delete-instance-profile --instance-profile-name "${iam_profile_name}"

    echo "Delete IAM role policy: ${iam_profile_name}"
    conceal-no-such-entity-response aws iam delete-role-policy --role-name "${iam_profile_name}" --policy-name "${iam_profile_name}"

    echo "Deleting IAM Role: ${iam_profile_name}"
    conceal-no-such-entity-response aws iam delete-role --role-name "${iam_profile_name}"
  done
}

# Detects NoSuchEntity response from AWS cli stderr output and conceals error
# Otherwise the error is treated as fatal
# usage: conceal-no-such-entity-response ...args
function conceal-no-such-entity-response {
  # in plain english: redirect stderr to stdout, and stdout to the log file
  local -r errMsg=$($@ 2>&1 > $LOG)
  if [[ "$errMsg" == "" ]];then
    return
  fi

  echo $errMsg
  if [[ "$errMsg" =~ " (NoSuchEntity) " ]];then
    echo " -> no such entity response detected. will assume operation is not necessary due to prior incomplete teardown"
    return
  fi

  echo "Error message is fatal. Will exit"
  exit 1
}

function ssh-key-setup {
  if [[ ! -f "$AWS_SSH_KEY" ]]; then
    ssh-keygen -f "$AWS_SSH_KEY" -N ''
  fi

  # Note that we use get-ssh-fingerprint, so this works on OSX Mavericks
  # get-aws-fingerprint gives the same fingerprint that AWS computes,
  # but OSX Mavericks ssh-keygen can't compute it
  AWS_SSH_KEY_FINGERPRINT=$(get-ssh-fingerprint ${AWS_SSH_KEY}.pub)
  echo "Using SSH key with (AWS) fingerprint: ${AWS_SSH_KEY_FINGERPRINT}"
  AWS_SSH_KEY_NAME="kubernetes-${AWS_SSH_KEY_FINGERPRINT//:/}"

  import-public-key ${AWS_SSH_KEY_NAME} ${AWS_SSH_KEY}.pub
}

function vpc-setup {
  if [[ -z "${VPC_ID:-}" ]]; then
    VPC_ID=$(get_vpc_id)
  fi
  if [[ -z "$VPC_ID" ]]; then
	  echo "Creating vpc."
	  VPC_ID=$($AWS_CMD create-vpc --cidr-block ${VPC_CIDR} --query Vpc.VpcId)
	  $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-support '{"Value": true}' > $LOG
	  $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-hostnames '{"Value": true}' > $LOG
	  add-tag $VPC_ID Name ${VPC_NAME}
	  add-tag $VPC_ID KubernetesCluster ${CLUSTER_ID}
  fi

  echo "Using VPC $VPC_ID"
}

function subnet-setup {
  if [[ -z "${SUBNET_ID:-}" ]]; then
    SUBNET_ID=$(get_subnet_id $VPC_ID $ZONE)
  fi

  if [[ -z "$SUBNET_ID" ]]; then
    echo "Creating subnet."
    SUBNET_ID=$($AWS_CMD create-subnet --cidr-block ${SUBNET_CIDR} --vpc-id $VPC_ID --availability-zone ${ZONE} --query Subnet.SubnetId)
    add-tag $SUBNET_ID KubernetesCluster ${CLUSTER_ID}
  else
    EXISTING_CIDR=$($AWS_CMD describe-subnets --subnet-ids ${SUBNET_ID} --query Subnets[].CidrBlock)
    echo "Using existing subnet with CIDR $EXISTING_CIDR"
    if [ ! $SUBNET_CIDR = $EXISTING_CIDR ]; then
      MASTER_INTERNAL_IP="${EXISTING_CIDR%.*}${MASTER_IP_SUFFIX}"
      echo "Assuming MASTER_INTERNAL_IP=${MASTER_INTERNAL_IP}"
    fi
  fi

  echo "Using subnet $SUBNET_ID"
}

function kube-up {
  echo "Starting cluster using os distro: ${OS_DISTRIBUTION}" >&2

  get-tokens

  detect-image
  detect-minion-image

  detect-root-device

  find-release-tars

  ensure-temp-dir

  create-bootstrap-script

  upload-server-tars

  ensure-iam-profiles

  load-or-gen-kube-basicauth
  load-or-gen-kube-bearertoken

  ssh-key-setup

  vpc-setup

  create-dhcp-option-set

  subnet-setup

  IGW_ID=$(get_igw_id $VPC_ID)
  if [[ -z "$IGW_ID" ]]; then
	  echo "Creating Internet Gateway."
	  IGW_ID=$($AWS_CMD create-internet-gateway --query InternetGateway.InternetGatewayId)
	  $AWS_CMD attach-internet-gateway --internet-gateway-id $IGW_ID --vpc-id $VPC_ID > $LOG
  fi

  echo "Using Internet Gateway $IGW_ID"

  echo "Associating route table."
  ROUTE_TABLE_ID=$($AWS_CMD describe-route-tables \
                            --filters Name=vpc-id,Values=${VPC_ID} \
                                      Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                            --query RouteTables[].RouteTableId)
  if [[ -z "${ROUTE_TABLE_ID}" ]]; then
    echo "Creating route table"
    ROUTE_TABLE_ID=$($AWS_CMD create-route-table \
                              --vpc-id=${VPC_ID} \
                              --query RouteTable.RouteTableId)
    add-tag ${ROUTE_TABLE_ID} KubernetesCluster ${CLUSTER_ID}
  fi

  echo "Associating route table ${ROUTE_TABLE_ID} to subnet ${SUBNET_ID}"
  $AWS_CMD associate-route-table --route-table-id $ROUTE_TABLE_ID --subnet-id $SUBNET_ID > $LOG || true
  echo "Adding route to route table ${ROUTE_TABLE_ID}"
  $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block 0.0.0.0/0 --gateway-id $IGW_ID > $LOG || true

  echo "Using Route Table $ROUTE_TABLE_ID"

  # Create security groups
  MASTER_SG_ID=$(get_security_group_id "${MASTER_SG_NAME}")
  if [[ -z "${MASTER_SG_ID}" ]]; then
    echo "Creating master security group."
    create-security-group "${MASTER_SG_NAME}" "Kubernetes security group applied to master nodes"
  fi
  NODE_SG_ID=$(get_security_group_id "${NODE_SG_NAME}")
  if [[ -z "${NODE_SG_ID}" ]]; then
    echo "Creating minion security group."
    create-security-group "${NODE_SG_NAME}" "Kubernetes security group applied to minion nodes"
  fi

  detect-security-groups

  # Masters can talk to master
  authorize-security-group-ingress "${MASTER_SG_ID}" "--source-group ${MASTER_SG_ID} --protocol all"

  # Minions can talk to minions
  authorize-security-group-ingress "${NODE_SG_ID}" "--source-group ${NODE_SG_ID} --protocol all"

  # Masters and minions can talk to each other
  authorize-security-group-ingress "${MASTER_SG_ID}" "--source-group ${NODE_SG_ID} --protocol all"
  authorize-security-group-ingress "${NODE_SG_ID}" "--source-group ${MASTER_SG_ID} --protocol all"

  # SSH is open to the world
  authorize-security-group-ingress "${MASTER_SG_ID}" "--protocol tcp --port 22 --cidr ${SSH_CIDR}"
  authorize-security-group-ingress "${NODE_SG_ID}" "--protocol tcp --port 22 --cidr ${SSH_CIDR}"

  # HTTPS to the master is allowed (for API access)
  authorize-security-group-ingress "${MASTER_SG_ID}" "--protocol tcp --port 443 --cidr ${HTTP_API_CIDR}"

  # KUBE_USE_EXISTING_MASTER is used to add minions to an existing master
  if [[ "${KUBE_USE_EXISTING_MASTER:-}" == "true" ]]; then
    detect-master
    parse-master-env

    # Start minions
    start-minions
    wait-minions
  else
    # Create the master
    start-master

    # Build ~/.kube/config
    build-config

    # Start minions
    start-minions
    wait-minions

    # Wait for the master to be ready
    wait-master
  fi

  # Check the cluster is OK
  check-cluster
}

# Builds the bootstrap script and saves it to a local temp file
# Sets BOOTSTRAP_SCRIPT to the path of the script
function create-bootstrap-script() {
  ensure-temp-dir

  BOOTSTRAP_SCRIPT="${KUBE_TEMP}/bootstrap-script"

  (
    # Include the default functions from the GCE configure-vm script
    sed '/^#+AWS_OVERRIDES_HERE/,$d' "${KUBE_ROOT}/cluster/gce/configure-vm.sh"
    # Include the AWS override functions
    cat "${KUBE_ROOT}/cluster/aws/templates/configure-vm-aws.sh"
    cat "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
    # Include the GCE configure-vm directly-executed code
    sed -e '1,/^#+AWS_OVERRIDES_HERE/d' "${KUBE_ROOT}/cluster/gce/configure-vm.sh"
  ) > "${BOOTSTRAP_SCRIPT}"
}

# Starts the master node
function start-master() {
  # Ensure RUNTIME_CONFIG is populated
  build-runtime-config

  # Get or create master persistent volume
  ensure-master-pd

  # Get or create master elastic IP
  ensure-master-ip

  # We have to make sure that the cert is valid for API_SERVERS
  # i.e. we likely have to pass ELB name / elastic IP in future
  create-certs "${KUBE_MASTER_IP}" "${MASTER_INTERNAL_IP}"

  # This key is no longer needed, and this enables us to get under the 16KB size limit
  KUBECFG_CERT_BASE64=""
  KUBECFG_KEY_BASE64=""

  write-master-env

  (
    # We pipe this to the ami as a startup script in the user-data field.  Requires a compatible ami
    echo "#! /bin/bash"
    echo "mkdir -p /var/cache/kubernetes-install"
    echo "cd /var/cache/kubernetes-install"

    echo "cat > kube_env.yaml << __EOF_MASTER_KUBE_ENV_YAML"
    cat ${KUBE_TEMP}/master-kube-env.yaml
    echo "AUTO_UPGRADE: 'true'"
    # TODO: get rid of these exceptions / harmonize with common or GCE
    echo "DOCKER_STORAGE: $(yaml-quote ${DOCKER_STORAGE:-})"
    echo "API_SERVERS: $(yaml-quote ${MASTER_INTERNAL_IP:-})"
    echo "__EOF_MASTER_KUBE_ENV_YAML"
    echo ""
    echo "wget -O bootstrap ${BOOTSTRAP_SCRIPT_URL}"
    echo "chmod +x bootstrap"
    echo "mkdir -p /etc/kubernetes"
    echo "mv kube_env.yaml /etc/kubernetes"
    echo "mv bootstrap /etc/kubernetes/"
    echo "cat > /etc/rc.local << EOF_RC_LOCAL"
    echo "#!/bin/sh -e"
    # We want to be sure that we don't pass an argument to bootstrap
    echo "/etc/kubernetes/bootstrap"
    echo "exit 0"
    echo "EOF_RC_LOCAL"
    echo "/etc/kubernetes/bootstrap"
  ) > "${KUBE_TEMP}/master-user-data"

  # Compress the data to fit under the 16KB limit (cloud-init accepts compressed data)
  gzip "${KUBE_TEMP}/master-user-data"

  echo "Starting Master"
  master_id=$($AWS_CMD run-instances \
    --image-id $AWS_IMAGE \
    --iam-instance-profile Name=$IAM_PROFILE_MASTER \
    --instance-type $MASTER_SIZE \
    --subnet-id $SUBNET_ID \
    --private-ip-address $MASTER_INTERNAL_IP \
    --key-name ${AWS_SSH_KEY_NAME} \
    --security-group-ids ${MASTER_SG_ID} \
    --associate-public-ip-address \
    --block-device-mappings "${MASTER_BLOCK_DEVICE_MAPPINGS}" \
    --user-data fileb://${KUBE_TEMP}/master-user-data.gz \
    --query Instances[].InstanceId)
  add-tag $master_id Name $MASTER_NAME
  add-tag $master_id Role $MASTER_TAG
  add-tag $master_id KubernetesCluster ${CLUSTER_ID}

  echo "Waiting for master to be ready"
  local attempt=0

  while true; do
    echo -n Attempt "$(($attempt+1))" to check for master node
    local ip=$(get_instance_public_ip ${master_id})
    if [[ -z "${ip}" ]]; then
      if (( attempt > 30 )); then
        echo
        echo -e "${color_red}master failed to start. Your cluster is unlikely" >&2
        echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2
        echo -e "cluster. (sorry!)${color_norm}" >&2
        exit 1
      fi
    else
      # We are not able to add an elastic ip, a route or volume to the instance until that instance is in "running" state.
      wait-for-instance-state ${master_id} "running"

      KUBE_MASTER=${MASTER_NAME}
      echo -e " ${color_green}[master running]${color_norm}"

      attach-ip-to-instance ${KUBE_MASTER_IP} ${master_id}

      # This is a race between instance start and volume attachment.  There appears to be no way to start an AWS instance with a volume attached.
      # To work around this, we wait for volume to be ready in setup-master-pd.sh
      echo "Attaching persistent data volume (${MASTER_DISK_ID}) to master"
      $AWS_CMD attach-volume --volume-id ${MASTER_DISK_ID} --device /dev/sdb --instance-id ${master_id}

      sleep 10
      $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MASTER_IP_RANGE} --instance-id $master_id > $LOG

      break
    fi
    echo -e " ${color_yellow}[master not working yet]${color_norm}"
    attempt=$(($attempt+1))
    sleep 10
  done
}

# Creates an ASG for the minion nodes
function start-minions() {
  # Minions don't currently use runtime config, but call it anyway for sanity
  build-runtime-config

  echo "Creating minion configuration"

  write-node-env

  (
    # We pipe this to the ami as a startup script in the user-data field.  Requires a compatible ami
    echo "#! /bin/bash"
    echo "mkdir -p /var/cache/kubernetes-install"
    echo "cd /var/cache/kubernetes-install"
    echo "cat > kube_env.yaml << __EOF_KUBE_ENV_YAML"
    cat ${KUBE_TEMP}/node-kube-env.yaml
    echo "AUTO_UPGRADE: 'true'"
    # TODO: get rid of these exceptions / harmonize with common or GCE
    echo "DOCKER_STORAGE: $(yaml-quote ${DOCKER_STORAGE:-})"
    echo "API_SERVERS: $(yaml-quote ${MASTER_INTERNAL_IP:-})"
    echo "__EOF_KUBE_ENV_YAML"
    echo ""
    echo "wget -O bootstrap ${BOOTSTRAP_SCRIPT_URL}"
    echo "chmod +x bootstrap"
    echo "mkdir -p /etc/kubernetes"
    echo "mv kube_env.yaml /etc/kubernetes"
    echo "mv bootstrap /etc/kubernetes/"
    echo "cat > /etc/rc.local << EOF_RC_LOCAL"
    echo "#!/bin/sh -e"
    # We want to be sure that we don't pass an argument to bootstrap
    echo "/etc/kubernetes/bootstrap"
    echo "exit 0"
    echo "EOF_RC_LOCAL"
    echo "/etc/kubernetes/bootstrap"
  ) > "${KUBE_TEMP}/node-user-data"

  # Compress the data to fit under the 16KB limit (cloud-init accepts compressed data)
  gzip "${KUBE_TEMP}/node-user-data"

  local public_ip_option
  if [[ "${ENABLE_NODE_PUBLIC_IP}" == "true" ]]; then
    public_ip_option="--associate-public-ip-address"
  else
    public_ip_option="--no-associate-public-ip-address"
  fi
  local spot_price_option
  if [[ -n "${NODE_SPOT_PRICE:-}" ]]; then
    spot_price_option="--spot-price ${NODE_SPOT_PRICE}"
  else
    spot_price_option=""
  fi
  ${AWS_ASG_CMD} create-launch-configuration \
      --launch-configuration-name ${ASG_NAME} \
      --image-id $KUBE_NODE_IMAGE \
      --iam-instance-profile ${IAM_PROFILE_NODE} \
      --instance-type $NODE_SIZE \
      --key-name ${AWS_SSH_KEY_NAME} \
      --security-groups ${NODE_SG_ID} \
      ${public_ip_option} \
      ${spot_price_option} \
      --block-device-mappings "${NODE_BLOCK_DEVICE_MAPPINGS}" \
      --user-data "fileb://${KUBE_TEMP}/node-user-data.gz"

  echo "Creating autoscaling group"
  ${AWS_ASG_CMD} create-auto-scaling-group \
      --auto-scaling-group-name ${ASG_NAME} \
      --launch-configuration-name ${ASG_NAME} \
      --min-size ${NUM_NODES} \
      --max-size ${NUM_NODES} \
      --vpc-zone-identifier ${SUBNET_ID} \
      --tags ResourceId=${ASG_NAME},ResourceType=auto-scaling-group,Key=Name,Value=${NODE_INSTANCE_PREFIX} \
             ResourceId=${ASG_NAME},ResourceType=auto-scaling-group,Key=Role,Value=${NODE_TAG} \
             ResourceId=${ASG_NAME},ResourceType=auto-scaling-group,Key=KubernetesCluster,Value=${CLUSTER_ID}
}

function wait-minions {
  # Wait for the minions to be running
  # TODO(justinsb): This is really not needed any more
  local attempt=0
  local max_attempts=30
  # Spot instances are slower to launch
  if [[ -n "${NODE_SPOT_PRICE:-}" ]]; then
    max_attempts=90
  fi
  while true; do
    detect-node-names > $LOG
    if [[ ${#NODE_IDS[@]} == ${NUM_NODES} ]]; then
      echo -e " ${color_green}${#NODE_IDS[@]} minions started; ready${color_norm}"
      break
    fi

    if (( attempt > max_attempts )); then
      echo
      echo "Expected number of minions did not start in time"
      echo
      echo -e "${color_red}Expected number of minions failed to start.  Your cluster is unlikely" >&2
      echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2
      echo -e "cluster. (sorry!)${color_norm}" >&2
      exit 1
    fi

    echo -e " ${color_yellow}${#NODE_IDS[@]} minions started; waiting${color_norm}"
    attempt=$(($attempt+1))
    sleep 10
  done
}

# Wait for the master to be started
function wait-master() {
  detect-master > $LOG

  echo "Waiting for cluster initialization."
  echo
  echo "  This will continually check to see if the API for kubernetes is reachable."
  echo "  This might loop forever if there was some uncaught error during start"
  echo "  up."
  echo

  until $(curl --insecure --user ${KUBE_USER}:${KUBE_PASSWORD} --max-time 5 \
    --fail --output $LOG --silent https://${KUBE_MASTER_IP}/healthz); do
    printf "."
    sleep 2
  done

  echo "Kubernetes cluster created."
}

# Creates the ~/.kube/config file, getting the information from the master
# The master must be running and set in KUBE_MASTER_IP
function build-config() {
  export KUBE_CERT="${CERT_DIR}/pki/issued/kubecfg.crt"
  export KUBE_KEY="${CERT_DIR}/pki/private/kubecfg.key"
  export CA_CERT="${CERT_DIR}/pki/ca.crt"
  export CONTEXT="${CONFIG_CONTEXT}"
  (
   umask 077

   # Update the user's kubeconfig to include credentials for this apiserver.
   create-kubeconfig

   create-kubeconfig-for-federation
  )
}

# Sanity check the cluster and print confirmation messages
function check-cluster() {
  echo "Sanity checking cluster..."

  sleep 5

  detect-nodes > $LOG

  # Don't bail on errors, we want to be able to print some info.
  set +e

  # Basic sanity checking
  # TODO(justinsb): This is really not needed any more
  local rc # Capture return code without exiting because of errexit bash option
  for (( i=0; i<${#KUBE_NODE_IP_ADDRESSES[@]}; i++)); do
      # Make sure docker is installed and working.
      local attempt=0
      while true; do
        local minion_ip=${KUBE_NODE_IP_ADDRESSES[$i]}
        echo -n "Attempt $(($attempt+1)) to check Docker on node @ ${minion_ip} ..."
        local output=`check-minion ${minion_ip}`
        echo $output
        if [[ "${output}" != "working" ]]; then
          if (( attempt > 20 )); then
            echo
            echo -e "${color_red}Your cluster is unlikely to work correctly." >&2
            echo "Please run ./cluster/kube-down.sh and re-create the" >&2
            echo -e "cluster. (sorry!)${color_norm}" >&2
            exit 1
          fi
        else
          break
        fi
        attempt=$(($attempt+1))
        sleep 30
      done
  done

  # ensures KUBECONFIG is set
  get-kubeconfig-basicauth
  echo
  echo -e "${color_green}Kubernetes cluster is running.  The master is running at:"
  echo
  echo -e "${color_yellow}  https://${KUBE_MASTER_IP}"
  echo
  echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}"
  echo
}

function kube-down {
  local vpc_id=$(get_vpc_id)
  if [[ -n "${vpc_id}" ]]; then
    local elb_ids=$(get_elbs_in_vpc ${vpc_id})
    if [[ -n "${elb_ids}" ]]; then
      echo "Deleting ELBs in: ${vpc_id}"
      for elb_id in ${elb_ids}; do
        aws elb delete-load-balancer --load-balancer-name=${elb_id} >$LOG
      done

      echo "Waiting for ELBs to be deleted"
      while true; do
        elb_ids=$(get_elbs_in_vpc ${vpc_id})
        if [[ -z "$elb_ids"  ]]; then
          echo "All ELBs deleted"
          break
        else
          echo "ELBs not yet deleted: $elb_ids"
          echo "Sleeping for 3 seconds..."
          sleep 3
        fi
      done
    fi

    if [[ -z "${KUBE_MASTER_ID-}" ]]; then
      KUBE_MASTER_ID=$(get_instanceid_from_name ${MASTER_NAME})
    fi
    if [[ -n "${KUBE_MASTER_ID-}" ]]; then
      delete-instance-alarms ${KUBE_MASTER_ID}
    fi

    echo "Deleting instances in VPC: ${vpc_id}"
    instance_ids=$($AWS_CMD describe-instances \
                            --filters Name=vpc-id,Values=${vpc_id} \
                                      Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                            --query Reservations[].Instances[].InstanceId)

    if [[ -n "${instance_ids}" ]]; then
      asg_groups=$($AWS_CMD   describe-instances \
                              --query 'Reservations[].Instances[].Tags[?Key==`aws:autoscaling:groupName`].Value[]' \
                              --instance-ids ${instance_ids})
      for asg_group in ${asg_groups}; do
        if [[ -n $(${AWS_ASG_CMD} describe-auto-scaling-groups --auto-scaling-group-names ${asg_group} --query AutoScalingGroups[].AutoScalingGroupName) ]]; then
          echo "Deleting auto-scaling group: ${asg_group}"
          ${AWS_ASG_CMD} delete-auto-scaling-group --force-delete --auto-scaling-group-name ${asg_group}
        fi
        if [[ -n $(${AWS_ASG_CMD} describe-launch-configurations --launch-configuration-names ${asg_group} --query LaunchConfigurations[].LaunchConfigurationName) ]]; then
          echo "Deleting auto-scaling launch configuration: ${asg_group}"
          ${AWS_ASG_CMD} delete-launch-configuration --launch-configuration-name ${asg_group}
        fi
      done

      $AWS_CMD terminate-instances --instance-ids ${instance_ids} > $LOG
      echo "Waiting for instances to be deleted"
      for instance_id in ${instance_ids}; do
        wait-for-instance-state ${instance_id} "terminated"
      done
      echo "All instances deleted"
    fi
    if [[ -n $(${AWS_ASG_CMD} describe-launch-configurations --launch-configuration-names ${ASG_NAME} --query LaunchConfigurations[].LaunchConfigurationName) ]]; then
      echo "Warning: default auto-scaling launch configuration ${ASG_NAME} still exists, attempting to delete"
      echo "  (This may happen if kube-up leaves just the launch configuration but no auto-scaling group.)"
      ${AWS_ASG_CMD} delete-launch-configuration --launch-configuration-name ${ASG_NAME} || true
    fi

    find-master-pd
    find-tagged-master-ip

    if [[ -n "${KUBE_MASTER_IP:-}" ]]; then
      release-elastic-ip ${KUBE_MASTER_IP}
    fi

    if [[ -n "${MASTER_DISK_ID:-}" ]]; then
      echo "Deleting volume ${MASTER_DISK_ID}"
      $AWS_CMD delete-volume --volume-id ${MASTER_DISK_ID} > $LOG
    fi

    echo "Cleaning up resources in VPC: ${vpc_id}"
    default_sg_id=$($AWS_CMD describe-security-groups \
                             --filters Name=vpc-id,Values=${vpc_id} \
                                       Name=group-name,Values=default \
                             --query SecurityGroups[].GroupId \
                    | tr "\t" "\n")
    sg_ids=$($AWS_CMD describe-security-groups \
                      --filters Name=vpc-id,Values=${vpc_id} \
                                Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                      --query SecurityGroups[].GroupId \
             | tr "\t" "\n")
    # First delete any inter-security group ingress rules
    # (otherwise we get dependency violations)
    for sg_id in ${sg_ids}; do
      # EC2 doesn't let us delete the default security group
      if [[ "${sg_id}" == "${default_sg_id}" ]]; then
        continue
      fi

      echo "Cleaning up security group: ${sg_id}"
      other_sgids=$(${AWS_CMD} describe-security-groups --group-id "${sg_id}" --query SecurityGroups[].IpPermissions[].UserIdGroupPairs[].GroupId)
      for other_sgid in ${other_sgids}; do
        $AWS_CMD revoke-security-group-ingress --group-id "${sg_id}" --source-group "${other_sgid}" --protocol all > $LOG
      done
    done

    for sg_id in ${sg_ids}; do
      # EC2 doesn't let us delete the default security group
      if [[ "${sg_id}" == "${default_sg_id}" ]]; then
        continue
      fi

      delete_security_group ${sg_id}
    done

    subnet_ids=$($AWS_CMD describe-subnets \
                          --filters Name=vpc-id,Values=${vpc_id} \
                                    Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                          --query Subnets[].SubnetId \
             | tr "\t" "\n")
    for subnet_id in ${subnet_ids}; do
      $AWS_CMD delete-subnet --subnet-id ${subnet_id} > $LOG
    done

    igw_ids=$($AWS_CMD describe-internet-gateways \
                       --filters Name=attachment.vpc-id,Values=${vpc_id} \
                       --query InternetGateways[].InternetGatewayId \
             | tr "\t" "\n")
    for igw_id in ${igw_ids}; do
      $AWS_CMD detach-internet-gateway --internet-gateway-id $igw_id --vpc-id $vpc_id > $LOG
      $AWS_CMD delete-internet-gateway --internet-gateway-id $igw_id > $LOG
    done

    route_table_ids=$($AWS_CMD describe-route-tables \
                               --filters Name=vpc-id,Values=$vpc_id \
                                         Name=route.destination-cidr-block,Values=0.0.0.0/0 \
                               --query RouteTables[].RouteTableId \
                      | tr "\t" "\n")
    for route_table_id in ${route_table_ids}; do
      $AWS_CMD delete-route --route-table-id $route_table_id --destination-cidr-block 0.0.0.0/0 > $LOG
    done
    route_table_ids=$($AWS_CMD describe-route-tables \
                               --filters Name=vpc-id,Values=$vpc_id \
                                         Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                               --query RouteTables[].RouteTableId \
                      | tr "\t" "\n")
    for route_table_id in ${route_table_ids}; do
      $AWS_CMD delete-route-table --route-table-id $route_table_id > $LOG
    done

    echo "Deleting VPC: ${vpc_id}"
    $AWS_CMD delete-vpc --vpc-id $vpc_id > $LOG
  else
    echo "" >&2
    echo -e "${color_red}Cluster NOT deleted!${color_norm}" >&2
    echo "" >&2
    echo "No VPC was found with tag KubernetesCluster=${CLUSTER_ID}" >&2
    echo "" >&2
    echo "If you are trying to delete a cluster in a shared VPC," >&2
    echo "please consider using one of the methods in the kube-deploy repo." >&2
    echo "See: https://github.com/kubernetes/kube-deploy/blob/master/docs/delete_cluster.md" >&2
    echo "" >&2
    echo "Note: You may be seeing this message may be because the cluster was already deleted, or" >&2
    echo "has a name other than '${CLUSTER_ID}'." >&2
  fi

  if [[ -z "${DHCP_OPTION_SET_ID:-}" ]]; then
    dhcp_option_ids=$($AWS_CMD describe-dhcp-options \
                               --output text \
                               --filters Name=tag:Name,Values=kubernetes-dhcp-option-set \
                                         Name=tag:KubernetesCluster,Values=${CLUSTER_ID} \
                               --query DhcpOptions[].DhcpOptionsId \
                      | tr "\t" "\n")
    for dhcp_option_id in ${dhcp_option_ids}; do
      echo "Deleting DHCP option set: ${dhcp_option_id}"
      $AWS_CMD delete-dhcp-options --dhcp-options-id $dhcp_option_id > $LOG
    done
  else
    echo "Skipping deletion of pre-existing DHCP option set: ${DHCP_OPTION_SET_ID}"
  fi

  echo "Deleting IAM Instance profiles"
  delete-iam-profiles
}

# Update a kubernetes cluster with latest source
function kube-push {
  detect-master

  # Make sure we have the tar files staged on Google Storage
  find-release-tars
  create-bootstrap-script
  upload-server-tars

  (
    echo "#! /bin/bash"
    echo "mkdir -p /var/cache/kubernetes-install"
    echo "cd /var/cache/kubernetes-install"
    echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
    echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh"
    echo "echo Executing configuration"
    echo "sudo salt '*' mine.update"
    echo "sudo salt --force-color '*' state.highstate"
  ) | ssh -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${KUBE_MASTER_IP} sudo bash

  get-kubeconfig-basicauth

  echo
  echo "Kubernetes cluster is running.  The master is running at:"
  echo
  echo "  https://${KUBE_MASTER_IP}"
  echo

}

# -----------------------------------------------------------------------------
# Cluster specific test helpers used from hack/e2e.go

# Execute prior to running tests to build a release if required for env.
#
# Assumed Vars:
#   KUBE_ROOT
function test-build-release {
  # Make a release
  "${KUBE_ROOT}/build/release.sh"
}

# Execute prior to running tests to initialize required structure. This is
# called from hack/e2e.go only when running -up.
#
# Assumed vars:
#   Variables from config.sh
function test-setup {
  "${KUBE_ROOT}/cluster/kube-up.sh"

  VPC_ID=$(get_vpc_id)
  detect-security-groups

  # Open up port 80 & 8080 so common containers on minions can be reached
  # TODO(roberthbailey): Remove this once we are no longer relying on hostPorts.
  authorize-security-group-ingress "${NODE_SG_ID}" "--protocol tcp --port 80 --cidr 0.0.0.0/0"
  authorize-security-group-ingress "${NODE_SG_ID}" "--protocol tcp --port 8080 --cidr 0.0.0.0/0"

  # Open up the NodePort range
  # TODO(justinsb): Move to main setup, if we decide whether we want to do this by default.
  authorize-security-group-ingress "${NODE_SG_ID}" "--protocol all --port 30000-32767 --cidr 0.0.0.0/0"

  echo "test-setup complete"
}

# Execute after running tests to perform any required clean-up. This is called
# from hack/e2e.go
function test-teardown {
  # (ingress rules will be deleted along with the security group)
  echo "Shutting down test cluster."
  "${KUBE_ROOT}/cluster/kube-down.sh"
}


# Gets the hostname (or IP) that we should SSH to for the given nodename
# For the master, we use the nodename, for the nodes we use their instanceids
function get_ssh_hostname {
  local node="$1"

  if [[ "${node}" == "${MASTER_NAME}" ]]; then
    node=$(get_instanceid_from_name ${MASTER_NAME})
    if [[ -z "${node-}" ]]; then
      echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'" 1>&2
      exit 1
    fi
  fi

  local ip=$(get_instance_public_ip ${node})
  if [[ -z "$ip" ]]; then
    echo "Could not detect IP for ${node}." 1>&2
    exit 1
  fi
  echo ${ip}
}

# SSH to a node by name ($1) and run a command ($2).
function ssh-to-node {
  local node="$1"
  local cmd="$2"

  local ip=$(get_ssh_hostname ${node})

  for try in {1..5}; do
    if ssh -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${ip} "echo test > /dev/null"; then
      break
    fi
    sleep 5
  done
  ssh -oLogLevel=quiet -oConnectTimeout=30 -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ${SSH_USER}@${ip} "${cmd}"
}

# Perform preparations required to run e2e tests
function prepare-e2e() {
  # (AWS runs detect-project, I don't think we need to anything)
  # Note: we can't print anything here, or else the test tools will break with the extra output
  return
}

function get-tokens() {
  KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
}