#!/bin/bash set -e export S3_BUCKET=${S3_BUCKET:-"pcd-community.s3-accelerate.amazonaws.com"} export S3_USER_AGENT=${S3_USER_AGENT:-""} export CALICO_INSTALL_TIMEOUT=${CALICO_INSTALL_TIMEOUT:-"900"} export STATE_FILE="$HOME/.airctl/state.yaml" # feature flag for audit components export ENABLE_AUDIT=${ENABLE_AUDIT:-"true"} # Whether this is a multi-region CE deployment. Left empty here and # populated by detect_multi_region() (from bork) before the upgrade is # triggered, so the multi_region option always matches the deployment. export MULTI_REGION=${MULTI_REGION:-""} # Verbose debug logging. Step banners are always shown; when DEBUG=true this # additionally dumps the rendered options.json and bork connection details. export DEBUG=${DEBUG:-"false"} # Capture whether the caller explicitly pinned a version, before the # ${PCD_VERSION:=...} default-assignment trick further down overwrites # this signal. Used both for the telemetry default below and for the # refork decision further down in this script. export PCD_VERSION_WAS_SET=${PCD_VERSION:+true} # Anonymous usage analytics are enabled by default to help improve the product. # To opt out, set TELEMETRY=false before running this script. Pinning a # specific PCD_VERSION (an older or unpublished build) defaults telemetry # off instead, since that's most likely a test/dev run. if [[ "$PCD_VERSION_WAS_SET" == "true" ]]; then export TELEMETRY=${TELEMETRY:-"false"} else export TELEMETRY=${TELEMETRY:-"true"} fi readFlag() { local longFlag="$1" local shortFlag="${2:-}" # Store all arguments in an array local args=("$@") # Skip the first two arguments (flag names) shift $(( $# > 1 ? 2 : 1 )) while [[ $# -gt 0 ]]; do case "$1" in --${longFlag}=*) # Handle --flag=value format echo "${1#*=}" return 0 ;; --${longFlag}) # Handle --flag value format shift echo "$1" return 0 ;; -${shortFlag}) # Handle -f value format shift echo "$1" return 0 ;; -${shortFlag}=*) # Handle -f=value format echo "${1#*=}" return 0 ;; esac shift done # Return empty string if flag not found echo "" return 1 } CONFIG=($(readFlag "config" "" "$@")) || CONFIG="${AIRCTL_CONFIG:-"/opt/pf9/airctl/conf/airctl-config.yaml"}" DU_FQDN=$(grep duFqdn ${CONFIG} | cut -d' ' -f2) REGIONS=($(grep duRegion ${CONFIG} | cut -d':' -f2)) SKIP_CLUSTER_UPGRADE=$(readFlag "skip-mgmt-plane" "" "$@") || SKIP_CLUSTER_UPGRADE="false" # Color codes RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' NC='\033[0m' # Spinner function if [[ $- == *x* ]]; then # For debug mode that shows all output run_with_spinner() { local message="$1" local commands="$2" echo "$message... processing" eval "$commands" local exit_status=$? if [ $exit_status -eq 0 ]; then echo " ${GREEN}Done${NC}" else echo " ${RED}Failed${NC}" exit $exit_status fi } else # For normal operation run_with_spinner() { local message="$1" local commands="$2" echo -n "$message... " # Create temporary files for stdout and stderr local stdout_log local stderr_log stdout_log=$(mktemp) stderr_log=$(mktemp) (eval "$commands") >"$stdout_log" 2>"$stderr_log" & local pid=$! # Simple blinking spinner while kill -0 $pid 2>/dev/null; do echo -ne "${RED}▓${NC}" sleep 0.5 echo -ne "\b${RED}▒${NC}" sleep 0.5 echo -ne "\b" done # Wait for process to finish and check its exit status if wait $pid; then exit_status=0 else exit_status=$? fi if [ $exit_status -eq 0 ]; then echo -e " ${GREEN}Done${NC}" else echo -e " ${RED}Failed${NC}" cat "$stderr_log" rm -f "$stdout_log" "$stderr_log" exit $exit_status fi # Clean up temporary files rm -f "$stdout_log" "$stderr_log" } fi # ── Logging helpers ───────────────────────────────────────────────────────── STEP=0 CURRENT_STEP="" COMPLETED_STEPS=() log_step() { [[ -n "$CURRENT_STEP" ]] && COMPLETED_STEPS+=("$CURRENT_STEP") STEP=$((STEP + 1)) CURRENT_STEP="[Step ${STEP}] $*" echo -e "\n${GREEN}==> ${CURRENT_STEP}${NC}" } # Progress tracking, reported by upgrade_summary() if the script exits non-zero. MGMT_PLANE_UPGRADED="false" MGMT_UPGRADED_REGIONS=() MGMT_CURRENT_REGION="" HOSTS_UPGRADED_REGIONS=() HOSTS_FAILED_REGIONS=() # On a failed exit, explain how far the upgrade got and how to resume. Not # inherited by subshells, so only the main script's exit triggers it. upgrade_summary() { local rc=$? [[ $rc -eq 0 ]] && return echo -e "\n${RED}==> Upgrade did not complete (exit code ${rc})${NC}" for s in "${COMPLETED_STEPS[@]}"; do echo -e " ${GREEN}✓${NC} ${s}" done [[ -n "$CURRENT_STEP" ]] && echo -e " ${RED}✗${NC} ${CURRENT_STEP}" [[ ${#MGMT_UPGRADED_REGIONS[@]} -gt 0 ]] && echo " Management plane upgraded for region(s): ${MGMT_UPGRADED_REGIONS[*]}" [[ -n "$MGMT_CURRENT_REGION" ]] && echo " Management plane upgrade failed for region: ${MGMT_CURRENT_REGION}" [[ ${#HOSTS_UPGRADED_REGIONS[@]} -gt 0 ]] && echo " Hosts upgraded for region(s): ${HOSTS_UPGRADED_REGIONS[*]}" [[ ${#HOSTS_FAILED_REGIONS[@]} -gt 0 ]] && echo " Host upgrade failed for region(s): ${HOSTS_FAILED_REGIONS[*]}" if [[ "$MGMT_PLANE_UPGRADED" == "true" || "$SKIP_CLUSTER_UPGRADE" != "false" ]]; then echo -e "\nThe management plane is upgraded. Retry the host upgrade with ${YELLOW}curl -sfL https://go.pcd.run/upgrade | bash -s -- --skip-mgmt-plane${NC}" else echo -e "\nThe management plane upgrade did not complete. Re-run the full upgrade with ${YELLOW}curl -sfL https://go.pcd.run/upgrade | bash${NC}" fi } trap upgrade_summary EXIT debug() { [[ "$DEBUG" == "true" ]] || return 0 echo -e "${YELLOW}[debug] $*${NC}" } # Dump the rendered options.json the upgrade will consume (pretty-printed via # python3, falling back to a raw cat). Gated on DEBUG. dump_options() { [[ "$DEBUG" == "true" ]] || return 0 local f="/opt/pf9/airctl/conf/options.json" echo -e "${YELLOW}[debug] rendered ${f}:${NC}" if [ -f "$f" ]; then python3 -m json.tool "$f" 2>/dev/null || cat "$f" else echo "(options.json not found)" fi } remove_hpas() { kubectl get hpa -A --no-headers | awk '{print "kubectl delete hpa " $2 " -n " $1 " && kubectl scale deployment " $2 " -n " $1 " --replicas=1"}' | bash } # GET a bork API path (relative to the API root) and echo the JSON response # body. Targets bork at its namespace-derived ingress host; -k because CE # serves it behind a self-signed cert. BORK_HOST and BORK_TOKEN are resolved # in detect_multi_region() before this is first called. bork_get() { curl -fsSk \ -H "Authorization: Bearer ${BORK_TOKEN}" \ -H "Accept: application/json" \ "https://${BORK_HOST}/bork/api/v1/$1" } # Install kubectl if it isn't already on PATH. detect_multi_region() needs it # to locate the bork namespace, and that runs before the main install step # (which also installs kubectl) below. ensure_kubectl() { if command -v kubectl >/dev/null 2>&1; then return fi run_with_spinner "Installing kubectl" " cd \${HOME}/pcd-ce && curl -LO \"https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl\" && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ " } BASE=$(echo $DU_FQDN | cut -d. -f1) DOMAIN=$(echo $DU_FQDN | cut -d. -f2-) BASE_LC=$(echo $BASE | tr '[:upper:]' '[:lower:]') DOMAIN_LC=$(echo $DOMAIN | tr '[:upper:]' '[:lower:]') detect_multi_region() { # Determine single vs multi-region CE deployment from the number of # regions bork knows about: 1 region => single-region, >1 => multi-region. local regions_json count # Resolve how to reach bork: the admin token from airctl state and the # bork namespace's ingress host. BORK_TOKEN=$(grep '^borkToken:' "${STATE_FILE}" 2>/dev/null | awk '{print $2}') if [ -z "$BORK_TOKEN" ]; then echo "Error: could not read borkToken from ${STATE_FILE}" exit 1 fi BORK_NS=$(kubectl get ns -o name 2>/dev/null | grep '\-bork$' | cut -d'/' -f2 | head -n1) if [ -z "$BORK_NS" ]; then echo "Error: could not find a -bork namespace in the cluster" exit 1 fi BORK_HOST="${BORK_NS}.${DOMAIN}" debug "bork host=${BORK_HOST} ns=${BORK_NS} token_present=$([ -n "$BORK_TOKEN" ] && echo yes || echo no)" # A failed bork call must stop the upgrade: guessing the topology or audit # state would re-render options.json wrongly for the existing DU. if ! regions_json=$(bork_get "regions/"); then echo "Error: bork regions/ request to https://${BORK_HOST} failed" exit 1 fi debug "bork regions/ response: ${regions_json}" # Parse with python3 (guaranteed on Ubuntu) instead of jq. count=$(echo "$regions_json" | python3 -c 'import sys, json; print(len(json.load(sys.stdin)["items"]))' 2>/dev/null) || count="" if ! [[ "$count" =~ ^[0-9]+$ ]] || [ "$count" -eq 0 ]; then echo "Error: could not determine region count from bork (count='${count}')" exit 1 fi if [ "$count" -gt 1 ]; then MULTI_REGION="true" else MULTI_REGION="false" fi # Derive audit state from the DU's existing bork options rather than the # local ENABLE_AUDIT default, so the upgrade preserves whatever the DU was # installed with. bork stores audit_disabled=true only when audit was # disabled at install; its absence means audit is enabled. Reuses the # regions_json already fetched above (no extra bork call). local audit_disabled audit_disabled=$(echo "$regions_json" | python3 -c 'import sys, json; items = json.load(sys.stdin).get("items", []); print("true" if any(str(i.get("options", {}).get("audit_disabled", "")).lower() == "true" for i in items) else "false")' 2>/dev/null) || audit_disabled="" if [ "$audit_disabled" = "true" ]; then ENABLE_AUDIT="false" elif [ "$audit_disabled" = "false" ]; then ENABLE_AUDIT="true" else echo "Error: could not determine audit state from bork regions" exit 1 fi debug "bork audit_disabled=${audit_disabled:-unknown} -> ENABLE_AUDIT=${ENABLE_AUDIT}" echo "✓ Detected ${count} region(s) -> multi_region=${MULTI_REGION}, audit_disabled=${audit_disabled:-unknown}" } echo "Private Cloud Director Community Edition Upgrade Started..." if [[ $SKIP_CLUSTER_UPGRADE != "false" ]]; then log_step "Skipping management plane upgrade (--skip-mgmt-plane)" run_with_spinner "Configuring Management Plane Pre-Upgrade Requirements" " remove_hpas " echo "Skipping Managment Plane Upgrade" else log_step "Resolving target version and downloading artifacts" if [[ -z "${_PCD_CE_BUNDLED:-}" ]]; then # Stable version run_with_spinner "Finding latest version" " rm -rf \${HOME}/pcd-ce && mkdir -p \${HOME}/pcd-ce && cd \${HOME}/pcd-ce && curl -O https://pcd-community.s3-accelerate.amazonaws.com/stable.txt " # If PCD_VERSION variable is set use that version instead of the stable STABLE_VERSION=$(cat $HOME/pcd-ce/stable.txt) export VERSION=${PCD_VERSION:=$STABLE_VERSION} # Download PCD CE artifacts run_with_spinner "Downloading artifacts" " cd \${HOME}/pcd-ce && curl \${S3_USER_AGENT} --silent https://\${S3_BUCKET}/\${VERSION}/index.txt | grep -e airctl -e install-pcd.sh -e pcd-chart.tgz -e options.json -e version.txt -e helm -e yq -e kaapi-chart.tgz -e ce-upgrade.sh | awk -v ver=\"\$VERSION\" -v s3b=\"\$S3_BUCKET\" -v s3ua=\"\$S3_USER_AGENT\" '{print \"curl \" s3ua \" -sS \\\"https://\" s3b \"/\" ver \"/\" \$NF \"\\\" -o \${HOME}/pcd-ce/\" \$NF}' | bash " if [[ "$PCD_VERSION_WAS_SET" == "true" && -f "$HOME/pcd-ce/ce-upgrade.sh" ]]; then chmod +x "$HOME/pcd-ce/ce-upgrade.sh" exec env _PCD_CE_BUNDLED=1 "$HOME/pcd-ce/ce-upgrade.sh" "$@" fi fi # Updating Chart URL while keeping the remaining options same as previous for upgrade # export chart_url=$(grep -o "\"chart_url\": \".*\"" ${HOME}/pcd-ce/options.json) # sed -i 's|"chart_url": ".*"|'"${chart_url}"'|' /opt/pf9/airctl/conf/options.json # sudo cp /opt/pf9/airctl/conf/options.json ${HOME}/pcd-ce/options.json # Detect single vs multi-region from the running DU's bork before the # upgrade is triggered, so the multi_region option below matches the # existing deployment. Needs kubectl, which the main install step below # also installs — ensure it's present first. log_step "Detecting region topology (single vs multi-region)" ensure_kubectl detect_multi_region log_step "Installing artifacts and rendering options.json" # Install PCD CE artifacts, modify options, and install kubectl run_with_spinner "Installing artifacts and dependencies" " cd ${HOME}/pcd-ce && chmod +x ./install-pcd.sh && ./install-pcd.sh ${VERSION} && sudo ln -sf /opt/pf9/airctl/airctl /usr/bin/airctl && # Value-agnostic overrides: match on the KEY, not the shipped default # value, so an override still lands if the bundled options.json changes # its defaults (e.g. skip_components already non-empty). See the # generated template in support/release/upload-pcd-artifacts.sh. if echo \"${ENABLE_AUDIT}\" | grep -qi '^true$'; then sed -i -E 's|\"skip_components\": \"[^\"]*\"|\"skip_components\": \"gnocchi,heat,terrakube,appcatalog\"|' /opt/pf9/airctl/conf/options.json else sed -i -E 's|\"skip_components\": \"[^\"]*\"|\"skip_components\": \"gnocchi,heat,audit,terrakube,appcatalog\"|' /opt/pf9/airctl/conf/options.json fi && sed -i -E 's|\"community_edition\": \"[^\"]*\"|\"community_edition\": \"true\"|' /opt/pf9/airctl/conf/options.json && sed -i -E 's|\"multi_region\": \"[^\"]*\"|\"multi_region\": \"${MULTI_REGION}\"|' /opt/pf9/airctl/conf/options.json && if echo \"${ENABLE_AUDIT}\" | grep -qiv '^true$'; then grep -q '\"audit_disabled\"' /opt/pf9/airctl/conf/options.json || sed -i -E 's|(\"community_edition\": \"[^\"]*\")|\1, \"audit_disabled\": \"true\"|' /opt/pf9/airctl/conf/options.json fi && # helm timeouts are absent from the shipped template, so sed cannot add # them; upsert via python so they are always present. python3 -c 'import json; f=\"/opt/pf9/airctl/conf/options.json\"; d=json.load(open(f)); d.update({\"helm_timeout\":\"30m0s\",\"helm_keystone_timeout\":\"30m0s\",\"helm_kubedu_timeout\":\"30m0s\"}); json.dump(d,open(f,\"w\"),indent=4)' && curl -LO \"https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl\" && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ " dump_options log_step "Applying pre-upgrade management plane requirements" run_with_spinner "Configuring Management Plane Pre-Upgrade Requirements" " remove_hpas " log_step "Triggering management plane upgrade" # Upgrade CE. For a multi-region deployment, upgrade the regions one at a # time (primary "Infra" first) via --region instead of a single all-regions # run, so each region is upgraded on its own and a not-yet-registered # secondary region can't stall the whole run. if [[ "${MULTI_REGION}" == "true" ]]; then UPGRADE_ORDER=() for R in "${REGIONS[@]}"; do [[ "$R" == "Infra" ]] && UPGRADE_ORDER+=("$R"); done for R in "${REGIONS[@]}"; do [[ "$R" != "Infra" ]] && UPGRADE_ORDER+=("$R"); done for R in "${UPGRADE_ORDER[@]}"; do echo -e "${GREEN}==> Upgrading management plane for region ${R}${NC}" MGMT_CURRENT_REGION="${R}" airctl upgrade --config ${CONFIG} --region "${R}" MGMT_UPGRADED_REGIONS+=("${R}") MGMT_CURRENT_REGION="" done else MGMT_CURRENT_REGION="${REGIONS[*]}" airctl upgrade --config ${CONFIG} MGMT_UPGRADED_REGIONS+=("${REGIONS[@]}") MGMT_CURRENT_REGION="" fi MGMT_PLANE_UPGRADED="true" echo "" airctl status --config ${CONFIG} fi # Upgrade Hosts wait_for_job() { JOB_NAME=$1 JOB_NS=$2 SUCCESS_MSG=$3 ERROR_MSG=$4 STREAM_FILE=$5 echo "" echo "Streaming logs for job \"$JOB_NAME\" in namespace \"$JOB_NS\" to \"$STREAM_FILE\"" kubectl get job "$JOB_NAME" -n $JOB_NS > /dev/null 2>&1 || { echo -e "❌ ${RED}${ERROR_MSG}: Failed to get job: \"$JOB_NAME\" in namespace \"$JOB_NS\"${NC}"; exit 1; } kubectl logs -f "job/$JOB_NAME" --all-containers -n $JOB_NS > "$STREAM_FILE" 2>&1 & LOG_PID=$! while true; do # Check job status COMPLETE_STATUS=$(kubectl get job "$JOB_NAME" -n $JOB_NS -o jsonpath='{.status.conditions[?(@.type=="Complete")].type}' || true) FAILED_STATUS=$(kubectl get job "$JOB_NAME" -n $JOB_NS -o jsonpath='{.status.conditions[?(@.type=="Failed")].type}' || true) if [ "$COMPLETE_STATUS" = "Complete" ]; then echo "" echo -e "✅ ${GREEN}${SUCCESS_MSG}: job \"$JOB_NAME\" in namespace \"$JOB_NS\" completed successfully${NC}" kill -9 $LOG_PID > /dev/null 2>&1 || true break elif [ "$FAILED_STATUS" = "Failed" ]; then echo "" echo -e "❌ ${RED}${ERROR_MSG}: job \"$JOB_NAME\" in namespace \"$JOB_NS\" failed${NC}" # Get failure message if available FAILURE_REASON=$(kubectl get job "$JOB_NAME" -n $JOB_NS -o jsonpath='{.status.conditions[?(@.type=="Failed")].message}' || true) [ -n "$FAILURE_REASON" ] && echo "Failure reason: $FAILURE_REASON" kill -9 $LOG_PID > /dev/null 2>&1 || true exit 1 fi # Wait before checking again sleep 5 done } # Runs in a subshell from the host loop, whose "if !" context disables set -e, # so every failure here must be handled explicitly with return/exit. send_upgrade_request() { REGION=$1 NS=$2 FQDN=$3 if ! /opt/pf9/airctl/airctl upgrade-hosts --config ${CONFIG} --region ${REGION}; then # The request never went out, so there is no job to watch. echo -e "❌ ${RED}Host upgrade request failed for region \"${REGION}\"${NC}" return 1 fi echo "" echo "Sent request to upgrade hosts for region: \"${REGION}\"" sleep 10 JOB_NAME=$(kubectl get jobs -n ${NS} --sort-by=.metadata.creationTimestamp | grep -E "host-upgrade-[0-9]+" | tac | head -n 1 | cut -d' ' -f1) if [ -z "$JOB_NAME" ]; then echo -e "❌ ${RED}No host-upgrade job found in namespace \"${NS}\" for region \"${REGION}\"${NC}" return 1 fi wait_for_job "${JOB_NAME}" "${NS}" "Host upgrade successful for region \"${REGION}\"" "Host upgrade failed for region \"${REGION}\"" "${NS}-upgrade.log" } echo -e "\nSleeping for 2 mins to make sure the management plane is ready..." sleep 120 # Ensure region topology is known before deciding whether to upgrade Infra. # The --skip-mgmt-plane path skips the detection done during the management # plane upgrade above, so MULTI_REGION may still be empty here. if [[ -z "${MULTI_REGION}" ]]; then ensure_kubectl detect_multi_region fi echo "" log_step "Upgrading hosts one region at a time" for REGION in "${REGIONS[@]}"; do REGION_LC=$(echo $REGION | tr '[:upper:]' '[:lower:]') # Skip Infra only for multi-region layouts, where it is control-plane-only. # In single-region CE, Infra also holds the workload hosts, so upgrade it. if [[ ${REGION} != "Infra" || "${MULTI_REGION}" == "false" ]]; then # In single-region CE, Infra is the whole DU and uses the base FQDN # (pcd.pf9.io) with no "-" suffix. Other regions get the suffix. if [[ ${REGION} == "Infra" ]]; then REGION_NS="${BASE_LC}" else REGION_NS="${BASE_LC}-${REGION_LC}" fi # Sequential: finish one region's host upgrade before starting the next # (no backgrounding). The subshell contains wait_for_job's "exit 1"; # a failed region is recorded and the remaining regions still run. if ( send_upgrade_request "${REGION}" "${REGION_NS}" "${REGION_NS}.${DOMAIN_LC}" ); then HOSTS_UPGRADED_REGIONS+=("${REGION}") else HOSTS_FAILED_REGIONS+=("${REGION}") fi fi done if [[ ${#HOSTS_FAILED_REGIONS[@]} -gt 0 ]]; then echo -e "\n❌ ${RED}One or more host upgrade jobs failed.${NC}" exit 1 else echo -e "\n${GREEN}✅ Host upgrade completed for all regions.${NC}" fi echo "" log_step "Fetching login credentials" # Get credentials and show output for REGION in "${REGIONS[@]}"; do if [[ ${REGION} != "Infra" || "${MULTI_REGION}" == "false" ]]; then REGION_LC=$(echo $REGION | tr '[:upper:]' '[:lower:]') echo "Login Details for region $REGION:" if [[ "${MULTI_REGION}" == "false" ]]; then echo "URL: https://$BASE_LC.$DOMAIN_LC" else echo "URL: https://$BASE_LC-$REGION_LC.$DOMAIN_LC" fi airctl get-creds --config ${CONFIG} echo "" fi done echo "" echo "See documentation at https://pcd.run for next steps." echo "" echo "Join the support community at https://reddit.com/r/platform9"