#!/usr/bin/env bash

set -euo pipefail
set -E

REPO_ROOT="/srv/capetownmarket-faas"
SERVICE_USER="capetownmarket"
ENV_FILE="/etc/capetownmarket-faas/capetownmarket-faas.env"
ARTIFACT=""
CONTROL_HOST="127.0.0.1"
COMPANY="CapeTownMarket"
RELEASE_NOTE="manual"
REQUESTED_COMMIT=""
WORKFLOW_RUN_ID=""
WORKFLOW_RUN_URL=""
PYTHON_VERSION="3.13"
UV_BIN="/usr/local/bin/uv"
NPM_BIN="npm"
SYSTEMCTL_BIN="systemctl"
NGINX_BIN="nginx"
NGINX_SITE_NAME="capetownmarket-faas"
NGINX_ERROR_PAGE_ROOT="/var/www/capetownmarket-faas-errors"
LOCK_FILE="/var/lock/capetownmarket-faas-deploy.lock"
RELEASE_STATE_ROOT="/var/lib/capetownmarket-faas/deploy-state"
EVIDENCE_DIR=""
SKIP_PORTAL_BUILD=0
WRITE_LOCK_MODE="drain"
WRITE_LOCK_REQUESTER="deploy-finance-faas-server"
DRAIN_TIMEOUT_SECONDS=300
DRAIN_INTERVAL_SECONDS=5
MAX_ACTIVE_SESSIONS=0
ACTIVE_SESSION_GRACE_SECONDS=0
CLOSE_ACTIVE_SESSIONS=0
WORK_DIR=""
BACKUP_ROOT=""
NEEDS_ROLLBACK=0
LOCK_HELD=0
WRITE_LOCK_CHANGED_BY_SCRIPT=0
DEPLOYMENT_MUTATION_STARTED=0
DEPLOYMENT_VERIFIED=0
DEPLOYMENT_REQUESTED_AT=""
DEPLOYMENT_STARTED_AT=""
DEPLOYMENT_COMPLETED_AT=""
LAST_ATTEMPT_RECORD=""
LAST_KNOWN_GOOD_RECORD=""
PREVIOUS_KNOWN_GOOD_COMMIT=""
PREVIOUS_KNOWN_GOOD_COMPLETED_AT=""
ROLLBACK_ATTEMPTED=0
ROLLBACK_VERIFIED=0
ROLLBACK_RESULT="not_attempted"
ROLLBACK_RESTORED_COMMIT=""
DEPLOYMENT_AUDIT_LOG=""
LAST_DEPLOYMENT_PUBLISH_KEY=""

log() {
  printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"
}

warn() {
  log "WARN: $*"
}

fail() {
  log "ERROR: $*"
  exit 1
}

bool_json() {
  if [[ "${1:-0}" -eq 1 ]]; then
    printf 'true'
  else
    printf 'false'
  fi
}

parse_bool_flag() {
  case "${1:-}" in
    1|true|TRUE|True|yes|YES|Yes|y|Y|on|ON|On)
      printf '1'
      ;;
    0|false|FALSE|False|no|NO|No|n|N|off|OFF|Off|"")
      printf '0'
      ;;
    *)
      fail "Expected a boolean value but received: ${1:-<empty>}"
      ;;
  esac
}

deployment_read_only_allowed() {
  local state="$1"
  local maintenance_pending="${2:-true}"

  case "$state" in
    normal|resumed|write_locked|draining)
      printf 'true'
      ;;
    failed)
      if [[ "$maintenance_pending" == "true" ]]; then
        printf 'false'
      else
        printf 'true'
      fi
      ;;
    *)
      printf 'false'
      ;;
  esac
}

deployment_block_new_logins() {
  local state="$1"
  local maintenance_pending="${2:-true}"

  case "$state" in
    normal|resumed)
      printf 'false'
      ;;
    failed)
      if [[ "$maintenance_pending" == "true" ]]; then
        printf 'true'
      else
        printf 'false'
      fi
      ;;
    *)
      printf 'true'
      ;;
  esac
}

usage() {
  cat <<'EOF'
Usage: deploy-finance-faas-server.sh --artifact /path/to/repo.tar [options]

Options:
  --repo-root PATH        Target repo root on the server (default: /srv/capetownmarket-faas)
  --env-file PATH         Shared EnvironmentFile path (default: /etc/capetownmarket-faas/capetownmarket-faas.env)
  --control-host HOST     Local host used for smoke checks (default: 127.0.0.1)
  --company NAME          Debtors/company smoke target (default: CapeTownMarket)
  --requested-at ISO8601  Deployment-cycle requested_at timestamp carried from workflow predeploy state
  --release-note TEXT     Free-form deploy note for logs (default: manual)
  --requested-commit SHA  Commit requested for deployment metadata
  --workflow-run-id ID    GitHub Actions run id for deployment metadata
  --workflow-run-url URL  GitHub Actions run URL for deployment metadata
  --service-user USER     Unix user that owns the runtime envs (default: capetownmarket)
  --python-version VER    Python version used by uv sync (default: 3.13)
  --uv-bin PATH           uv binary path (default: /usr/local/bin/uv)
  --npm-bin PATH          npm binary path (default: npm)
  --systemctl PATH        systemctl binary path (default: systemctl)
  --nginx-bin PATH        nginx binary path (default: nginx)
  --lock-file PATH        Deploy lock file path (default: /var/lock/capetownmarket-faas-deploy.lock)
  --release-state-root PATH
                         Persistent server-side deploy state root
                         (default: /var/lib/capetownmarket-faas/deploy-state)
  --evidence-dir PATH     Directory where deploy evidence will be written
  --skip-portal-build     Preserve the current built portal assets and skip
                         the portal-web npm install/build step
  --write-lock-mode MODE  Deployment write-lock mode: off or drain (default: drain)
  --write-lock-requester  Identity recorded in system write-lock state (default: deploy-finance-faas-server)
  --drain-timeout SEC     Max seconds to wait for a safe deploy boundary (default: 300)
                         Set to 0 to skip waiting and continue immediately
  --drain-interval SEC    Seconds between drain-state polls (default: 5)
  --max-active-sessions N Maximum active sessions allowed before deploy continues (default: 0)
  --active-session-grace-seconds SEC
                         Seconds to require zero active sessions before allowing the
                         configured --max-active-sessions threshold (default: 0)
  --close-active-sessions BOOL
                         Close active runtime sessions before deploy continues
                         (default: false)
EOF
}

escape_sed_replacement() {
  printf '%s' "$1" | sed 's/[&|]/\\&/g'
}

require_command() {
  local name="$1"
  command -v "$name" >/dev/null 2>&1 || fail "Required command not found: $name"
}

require_executable() {
  local path_or_name="$1"
  if [[ "$path_or_name" == */* ]]; then
    [[ -x "$path_or_name" ]] || fail "Required executable not found or not executable: $path_or_name"
    return 0
  fi
  require_command "$path_or_name"
}

run_as_service_user() {
  local command_text="$1"
  local home_dir="/home/$SERVICE_USER"
  command -v runuser >/dev/null 2>&1 || fail "runuser is required but not available."
  runuser -u "$SERVICE_USER" -- env -i \
    HOME="$home_dir" \
    USER="$SERVICE_USER" \
    LOGNAME="$SERVICE_USER" \
    PATH="/usr/local/bin:/usr/bin:/bin" \
    bash -lc "$command_text"
}

stop_service() {
  local unit="$1"
  if ! "$SYSTEMCTL_BIN" list-unit-files "$unit" >/dev/null 2>&1; then
    log "Skipping stop for $unit (unit not installed)"
    return 0
  fi
  if "$SYSTEMCTL_BIN" is-active --quiet "$unit"; then
    log "Stopping $unit"
    "$SYSTEMCTL_BIN" stop "$unit"
    return 0
  fi
  log "$unit already stopped"
}

start_service() {
  local unit="$1"
  log "Starting $unit"
  "$SYSTEMCTL_BIN" start "$unit"
}

wait_for_json() {
  local url="$1"
  local jq_filter="$2"
  local timeout_seconds="${3:-60}"
  local interval_seconds="${4:-2}"
  local deadline
  deadline=$((SECONDS + timeout_seconds))

  while (( SECONDS < deadline )); do
    if curl -fsS "$url" | jq -e "$jq_filter" >/dev/null; then
      return 0
    fi
    sleep "$interval_seconds"
  done

  return 1
}

wait_for_http() {
  local url="$1"
  local timeout_seconds="${2:-60}"
  local interval_seconds="${3:-2}"
  local deadline
  deadline=$((SECONDS + timeout_seconds))

  while (( SECONDS < deadline )); do
    if curl -fsS "$url" >/dev/null; then
      return 0
    fi
    sleep "$interval_seconds"
  done

  return 1
}

timestamp_now() {
  date -u --iso-8601=seconds
}

control_api_url() {
  printf 'http://%s:50102%s' "$CONTROL_HOST" "$1"
}

system_write_lock_enabled() {
  curl -fsS "$(control_api_url "/system/write-lock")" |
    jq -e '.system_write_lock.enabled == true' >/dev/null
}

system_write_lock_owned_by_deploy() {
  curl -fsS "$(control_api_url "/system/write-lock")" |
    jq -e --arg requester "$WRITE_LOCK_REQUESTER" \
      '.system_write_lock.enabled == true and .system_write_lock.enabled_by == $requester' >/dev/null
}

request_system_write_lock_state() {
  local enabled_literal="$1"
  local reason="$2"
  local message_id payload jq_filter

  message_id="deploy-write-lock-$(date +%s)-$RANDOM"
  payload="$(
    jq -n \
      --arg message_id "$message_id" \
      --arg requester "$WRITE_LOCK_REQUESTER" \
      --arg reason "$reason" \
      --arg trace_id "$message_id" \
      --argjson enabled "$enabled_literal" \
      '{
        message_id: $message_id,
        tenant_id: "system",
        session_type: "system",
        actor: $requester,
        intent: "request_system_write_lock",
        payload: {
          enabled: $enabled,
          requested_by: $requester,
          reason: $reason
        },
        routing_key: "system",
        synchronous_required: true,
        sender_queue: "sender",
        trace_id: $trace_id,
        metadata: {
          source: "deploy-finance-faas-server"
        }
      }'
  )"
  curl -fsS \
    -X POST \
    -H "Content-Type: application/json" \
    -d "$payload" \
    "$(control_api_url "/ingest")" >/dev/null

  if [[ "$enabled_literal" == "true" ]]; then
    jq_filter='.system_write_lock.enabled == true'
  else
    jq_filter='.system_write_lock.enabled == false'
  fi
  if ! wait_for_json "$(control_api_url "/system/write-lock")" "$jq_filter" 30 1; then
    warn "System write lock did not reach expected state ($enabled_literal)."
    return 1
  fi
}

request_deployment_state() {
  local state="$1"
  local result="${2:-}"
  local failure_reason="${3:-}"
  local maintenance_pending="${4:-true}"
  local status_detail="${5:-}"
  local read_only_allowed="${6:-$(deployment_read_only_allowed "$state" "$maintenance_pending")}"
  local block_new_logins="${7:-$(deployment_block_new_logins "$state" "$maintenance_pending")}"
  local message_id payload jq_filter

  message_id="deploy-state-$(date +%s)-$RANDOM"
  payload="$(
    jq -n \
      --arg message_id "$message_id" \
      --arg requester "$WRITE_LOCK_REQUESTER" \
      --arg state "$state" \
      --arg requested_at "$DEPLOYMENT_REQUESTED_AT" \
      --arg started_at "$DEPLOYMENT_STARTED_AT" \
      --arg completed_at "$DEPLOYMENT_COMPLETED_AT" \
      --arg commit "$REQUESTED_COMMIT" \
      --arg release_note "$RELEASE_NOTE" \
      --arg workflow_run_id "$WORKFLOW_RUN_ID" \
      --arg workflow_run_url "$WORKFLOW_RUN_URL" \
      --arg result "$result" \
      --arg failure_reason "$failure_reason" \
      --arg status_detail "$status_detail" \
      --arg trace_id "$message_id" \
      --argjson maintenance_pending "$maintenance_pending" \
      --argjson read_only_allowed "$read_only_allowed" \
      --argjson block_new_logins "$block_new_logins" \
      '
        def empty_to_null:
          if . == "" then null else . end;
        {
          message_id: $message_id,
          tenant_id: "system",
          session_type: "system",
          actor: $requester,
          intent: "request_deployment_state",
          payload: {
            state: $state,
            requested_by: $requester,
            requested_at: ($requested_at | empty_to_null),
            started_at: ($started_at | empty_to_null),
            completed_at: ($completed_at | empty_to_null),
            commit: ($commit | empty_to_null),
            release_note: ($release_note | empty_to_null),
            workflow_run_id: ($workflow_run_id | empty_to_null),
            workflow_run_url: ($workflow_run_url | empty_to_null),
            result: ($result | empty_to_null),
            failure_reason: ($failure_reason | empty_to_null),
            status_detail: ($status_detail | empty_to_null),
            maintenance_pending: $maintenance_pending,
            read_only_allowed: $read_only_allowed,
            block_new_logins: $block_new_logins
          },
          routing_key: "system",
          synchronous_required: true,
          sender_queue: "sender",
          trace_id: $trace_id,
          metadata: {
            source: "deploy-finance-faas-server"
          }
        }
      '
  )"

  curl -fsS \
    -X POST \
    -H "Content-Type: application/json" \
    -d "$payload" \
    "$(control_api_url "/ingest")" >/dev/null

  jq_filter=".deployment_state.state == \"$state\""
  wait_for_json "$(control_api_url "/system/deployment")" "$jq_filter" 30 1
}

append_deploy_audit_record() {
  local event_type="$1"
  local state="${2:-}"
  local result="${3:-}"
  local failure_reason="${4:-}"
  local maintenance_pending="${5:-false}"
  local status_detail="${6:-}"
  local read_only_allowed="${7:-true}"
  local block_new_logins="${8:-false}"

  [[ -n "$DEPLOYMENT_AUDIT_LOG" ]] || return 0
  mkdir -p "$(dirname "$DEPLOYMENT_AUDIT_LOG")"

  jq -nc \
    --arg event_at "$(timestamp_now)" \
    --arg event_type "$event_type" \
    --arg state "$state" \
    --arg release_note "$RELEASE_NOTE" \
    --arg requested_commit "$REQUESTED_COMMIT" \
    --arg requested_at "$DEPLOYMENT_REQUESTED_AT" \
    --arg started_at "$DEPLOYMENT_STARTED_AT" \
    --arg completed_at "$DEPLOYMENT_COMPLETED_AT" \
    --arg workflow_run_id "$WORKFLOW_RUN_ID" \
    --arg workflow_run_url "$WORKFLOW_RUN_URL" \
    --arg result "$result" \
    --arg failure_reason "$failure_reason" \
    --arg status_detail "$status_detail" \
    --arg previous_known_good_commit "$PREVIOUS_KNOWN_GOOD_COMMIT" \
    --arg previous_known_good_completed_at "$PREVIOUS_KNOWN_GOOD_COMPLETED_AT" \
    --arg rollback_result "$ROLLBACK_RESULT" \
    --arg rollback_restored_commit "$ROLLBACK_RESTORED_COMMIT" \
    --argjson maintenance_pending "$maintenance_pending" \
    --argjson read_only_allowed "$read_only_allowed" \
    --argjson block_new_logins "$block_new_logins" \
    --argjson rollback_attempted "$(bool_json "$ROLLBACK_ATTEMPTED")" \
    --argjson rollback_verified "$(bool_json "$ROLLBACK_VERIFIED")" \
    '
      def empty_to_null:
        if . == "" then null else . end;
      {
        event_at: ($event_at | empty_to_null),
        event_type: $event_type,
        state: ($state | empty_to_null),
        release_note: ($release_note | empty_to_null),
        requested_commit: ($requested_commit | empty_to_null),
        requested_at: ($requested_at | empty_to_null),
        started_at: ($started_at | empty_to_null),
        completed_at: ($completed_at | empty_to_null),
        workflow_run_id: ($workflow_run_id | empty_to_null),
        workflow_run_url: ($workflow_run_url | empty_to_null),
        result: ($result | empty_to_null),
        failure_reason: ($failure_reason | empty_to_null),
        status_detail: ($status_detail | empty_to_null),
        maintenance_pending: $maintenance_pending,
        read_only_allowed: $read_only_allowed,
        block_new_logins: $block_new_logins,
        previous_known_good_commit: ($previous_known_good_commit | empty_to_null),
        previous_known_good_completed_at: ($previous_known_good_completed_at | empty_to_null),
        rollback_attempted: $rollback_attempted,
        rollback_verified: $rollback_verified,
        rollback_result: ($rollback_result | empty_to_null),
        rollback_restored_commit: ($rollback_restored_commit | empty_to_null)
      }
    ' >>"$DEPLOYMENT_AUDIT_LOG"

  chmod a+r "$DEPLOYMENT_AUDIT_LOG" || true
}

update_deployment_state() {
  local state="$1"
  local result="${2:-}"
  local failure_reason="${3:-}"
  local maintenance_pending="${4:-true}"
  local status_detail="${5:-}"
  local read_only_allowed="${6:-$(deployment_read_only_allowed "$state" "$maintenance_pending")}"
  local block_new_logins="${7:-$(deployment_block_new_logins "$state" "$maintenance_pending")}"
  local publish_key
  local now

  now="$(timestamp_now)"
  if [[ -z "$DEPLOYMENT_REQUESTED_AT" ]]; then
    DEPLOYMENT_REQUESTED_AT="$now"
  fi
  if [[ -z "$DEPLOYMENT_STARTED_AT" ]] &&
     [[ "$state" == "deploying" || "$state" == "verifying" || "$state" == "failed" || "$state" == "resumed" ]]; then
    DEPLOYMENT_STARTED_AT="$now"
  fi
  if [[ -z "$DEPLOYMENT_COMPLETED_AT" ]] &&
     [[ "$state" == "failed" || "$state" == "resumed" ]]; then
    DEPLOYMENT_COMPLETED_AT="$now"
  fi

  publish_key="$state|$result|$failure_reason|$maintenance_pending|$status_detail|$read_only_allowed|$block_new_logins"
  if [[ "$publish_key" == "$LAST_DEPLOYMENT_PUBLISH_KEY" ]]; then
    return 0
  fi

  if ! request_deployment_state "$state" "$result" "$failure_reason" "$maintenance_pending" "$status_detail" "$read_only_allowed" "$block_new_logins"; then
    warn "Unable to publish deployment state '$state'"
    append_deploy_audit_record "deployment_state_publish_failed" "$state" "$result" "$failure_reason" "$maintenance_pending" "$status_detail" "$read_only_allowed" "$block_new_logins"
    return 0
  fi

  LAST_DEPLOYMENT_PUBLISH_KEY="$publish_key"
  append_deploy_audit_record "deployment_state" "$state" "$result" "$failure_reason" "$maintenance_pending" "$status_detail" "$read_only_allowed" "$block_new_logins"
}

enable_system_write_lock() {
  if [[ "$WRITE_LOCK_MODE" == "off" ]]; then
    log "Write-lock gate disabled for this deployment"
    return 0
  fi
  if system_write_lock_enabled; then
    if system_write_lock_owned_by_deploy; then
      log "System write lock already enabled by deploy controller; adopting it for this deployment"
      WRITE_LOCK_CHANGED_BY_SCRIPT=1
    else
      log "System write lock already enabled; leaving the existing operator state in place"
    fi
    return 0
  fi
  log "Enabling system write lock"
  request_system_write_lock_state "true" "Deployment requested: ${RELEASE_NOTE}" ||
    fail "Unable to enable the system write lock."
  WRITE_LOCK_CHANGED_BY_SCRIPT=1
  update_deployment_state "write_locked" "" "" "true" \
    "Writes are paused while deployment waits for a safe drain boundary."
}

disable_system_write_lock() {
  local reason="${1:-Deployment completed: ${RELEASE_NOTE}}"
  if [[ "$WRITE_LOCK_CHANGED_BY_SCRIPT" -ne 1 ]]; then
    return 0
  fi
  log "Clearing system write lock"
  request_system_write_lock_state "false" "$reason" || return 1
  WRITE_LOCK_CHANGED_BY_SCRIPT=0
}

count_sender_pending_messages() {
  curl -fsS "$(control_api_url "/sender/queue")" |
    jq -r '[.messages[]? | select(((.completion_status // "") | tostring) as $s | ["sent", "failed"] | index($s) | not)] | length'
}

count_active_operations() {
  curl -fsS "$(control_api_url "/operations/queue?limit=500")" |
    jq -r '[.operations[]? | select(((.status // "") | tostring) == "running")] | length'
}

count_active_work_items() {
  curl -fsS "$(control_api_url "/work-items/queue?limit=500")" |
    jq -r '[.items[]? | select(((.status // "") | tostring) as $s | ["queued", "running", "waiting", "retry_wait"] | index($s))] | length'
}

count_active_print_jobs() {
  curl -fsS "$(control_api_url "/print/jobs/queue?limit=500")" |
    jq -r '[.jobs[]? | select(((.status // "") | tostring) as $s | ["queued", "rendering", "retry_wait"] | index($s))] | length'
}

count_active_sessions() {
  curl -fsS "$(control_api_url "/state")" | jq -r '.active_session_count // 0'
}

close_active_sessions_for_deploy() {
  local active_tokens_json requested_count close_summary closed_count status_detail

  if (( CLOSE_ACTIVE_SESSIONS != 1 )); then
    return 0
  fi

  active_tokens_json="$(curl -fsS "$(control_api_url "/system/shutdown")" | jq -c '[.system_shutdown.active_sessions[]? | .session_token | select(type == "string" and length > 0)]')" || {
    warn "Could not fetch active sessions for predeploy cleanup; continuing."
    return 0
  }

  requested_count="$(printf '%s' "$active_tokens_json" | jq -r 'length')"
  if (( requested_count == 0 )); then
    log "No active sessions need to be closed before deployment."
    return 0
  fi

  status_detail="Closing ${requested_count} active session(s) before deployment continues."
  log "$status_detail"
  update_deployment_state "draining" "" "" "true" "$status_detail"

  close_summary="$(ACTIVE_SESSION_TOKENS_JSON="$active_tokens_json" python3 - "$REPO_ROOT" <<'PY'
import json
import os
import sqlite3
import sys
from datetime import datetime, timezone
from pathlib import Path

tokens = json.loads(os.environ.get("ACTIVE_SESSION_TOKENS_JSON", "[]"))
normalized = []
for token in tokens:
    if not isinstance(token, str):
        continue
    value = token.strip()
    if not value:
        continue
    normalized.append(value.removeprefix("session:"))

state_db = (
    Path(sys.argv[1])
    / "services"
    / "capetown-faas-core"
    / ".faas-state"
    / "runtime-state.db"
)

if not state_db.exists():
    print(
        json.dumps(
            {
                "requested": len(normalized),
                "closed": 0,
                "tokens": [],
                "state_db_exists": False,
            }
        )
    )
    raise SystemExit(0)

closed = []
now = datetime.now(timezone.utc).isoformat()
with sqlite3.connect(state_db) as conn:
    for token in normalized:
        updated = conn.execute(
            "UPDATE sessions SET status='closed', last_seen_at=? WHERE session_token=? AND status='active'",
            (now, token),
        ).rowcount
        if updated:
            closed.append(token)
    conn.commit()

print(
    json.dumps(
        {
            "requested": len(normalized),
            "closed": len(closed),
            "tokens": closed,
            "state_db_exists": True,
        }
    )
)
PY
)" || {
    warn "Could not close active sessions for deployment; continuing."
    return 0
  }

  closed_count="$(printf '%s' "$close_summary" | jq -r '.closed // 0')"
  status_detail="Closed ${closed_count} active session(s) before maintenance."
  log "$status_detail"
  update_deployment_state "draining" "" "" "true" "$status_detail"
  append_deploy_audit_record "active_sessions_closed" "draining" "" "" "true" "$status_detail" "true" "true"
}

wait_for_safe_deploy_boundary() {
  local deadline grace_deadline active_sessions sender_pending operations_active
  local work_items_active print_jobs_active effective_max_active_sessions
  local session_policy_note status_detail

  if [[ "$WRITE_LOCK_MODE" == "off" ]]; then
    return 0
  fi

  if (( DRAIN_TIMEOUT_SECONDS <= 0 )); then
    status_detail="Drain wait skipped because FINANCE_FAAS_DRAIN_TIMEOUT_SECONDS is set to ${DRAIN_TIMEOUT_SECONDS}s."
    log "$status_detail"
    update_deployment_state "draining" "" "" "true" "$status_detail"
    append_deploy_audit_record "safe_boundary_skipped" "draining" "" "" "true" "$status_detail" "true" "true"
    return 0
  fi

  update_deployment_state "draining" "" "" "true" \
    "Waiting for in-flight work to drain before maintenance begins."
  deadline=$((SECONDS + DRAIN_TIMEOUT_SECONDS))
  grace_deadline=$((SECONDS + ACTIVE_SESSION_GRACE_SECONDS))
  while (( SECONDS < deadline )); do
    active_sessions="$(count_active_sessions)"
    sender_pending="$(count_sender_pending_messages)"
    operations_active="$(count_active_operations)"
    work_items_active="$(count_active_work_items)"
    print_jobs_active="$(count_active_print_jobs)"

    effective_max_active_sessions=$MAX_ACTIVE_SESSIONS
    session_policy_note="strict_max=${MAX_ACTIVE_SESSIONS}"
    if (( ACTIVE_SESSION_GRACE_SECONDS > 0 )) && (( MAX_ACTIVE_SESSIONS > 0 )); then
      session_policy_note="threshold_max=${MAX_ACTIVE_SESSIONS}"
      if (( SECONDS < grace_deadline )); then
        effective_max_active_sessions=0
        session_policy_note="grace_zero_until=${ACTIVE_SESSION_GRACE_SECONDS}s"
      fi
    fi

    log "Drain status: sessions=$active_sessions session_limit=$effective_max_active_sessions session_policy=$session_policy_note sender_pending=$sender_pending operations=$operations_active work_items=$work_items_active print_jobs=$print_jobs_active"

    status_detail="Waiting on sessions=${active_sessions}/${effective_max_active_sessions} (${session_policy_note}), sender=${sender_pending}, operations=${operations_active}, work_items=${work_items_active}, print_jobs=${print_jobs_active}."
    update_deployment_state "draining" "" "" "true" "$status_detail"

    if (( active_sessions <= effective_max_active_sessions )) &&
       (( sender_pending == 0 )) &&
       (( operations_active == 0 )) &&
       (( work_items_active == 0 )) &&
       (( print_jobs_active == 0 )); then
      log "Safe deploy boundary reached"
      append_deploy_audit_record "safe_boundary_reached" "draining" "" "" "true" "$status_detail" "true" "true"
      return 0
    fi

    sleep "$DRAIN_INTERVAL_SECONDS"
  done

  return 1
}

acquire_deploy_lock() {
  mkdir -p "$(dirname "$LOCK_FILE")"
  exec 9>"$LOCK_FILE"
  if ! flock -n 9; then
    fail "Another deployment is already running (lock: $LOCK_FILE)"
  fi
  LOCK_HELD=1
  log "Acquired deploy lock at $LOCK_FILE"
}

release_deploy_lock() {
  if [[ "$LOCK_HELD" -ne 1 ]]; then
    return 0
  fi
  flock -u 9 || true
  exec 9>&-
  LOCK_HELD=0
  log "Released deploy lock at $LOCK_FILE"
}

ensure_release_state_paths() {
  local default_evidence_slug

  mkdir -p "$RELEASE_STATE_ROOT"
  LAST_ATTEMPT_RECORD="$RELEASE_STATE_ROOT/last_attempt.json"
  LAST_KNOWN_GOOD_RECORD="$RELEASE_STATE_ROOT/last_known_good.json"
  DEPLOYMENT_AUDIT_LOG="$RELEASE_STATE_ROOT/deploy-history.jsonl"
  touch "$DEPLOYMENT_AUDIT_LOG"
  chmod a+r "$DEPLOYMENT_AUDIT_LOG" || true

  if [[ -f "$LAST_KNOWN_GOOD_RECORD" ]]; then
    PREVIOUS_KNOWN_GOOD_COMMIT="$(jq -r '.commit // ""' "$LAST_KNOWN_GOOD_RECORD" 2>/dev/null || printf '')"
    PREVIOUS_KNOWN_GOOD_COMPLETED_AT="$(jq -r '.completed_at // ""' "$LAST_KNOWN_GOOD_RECORD" 2>/dev/null || printf '')"
  fi

  if [[ -z "$EVIDENCE_DIR" ]]; then
    default_evidence_slug="${WORKFLOW_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}"
    EVIDENCE_DIR="$RELEASE_STATE_ROOT/evidence/$default_evidence_slug"
  fi
  mkdir -p "$EVIDENCE_DIR"
}

write_release_record() {
  local destination="$1"
  local record_type="$2"
  local result="$3"
  local failure_reason="${4:-}"
  local tmp_file rollback_attempted rollback_verified

  tmp_file="$(mktemp)"
  rollback_attempted="$(bool_json "$ROLLBACK_ATTEMPTED")"
  rollback_verified="$(bool_json "$ROLLBACK_VERIFIED")"

  jq -n \
    --arg record_type "$record_type" \
    --arg release_note "$RELEASE_NOTE" \
    --arg requested_commit "$REQUESTED_COMMIT" \
    --arg requested_at "$DEPLOYMENT_REQUESTED_AT" \
    --arg started_at "$DEPLOYMENT_STARTED_AT" \
    --arg completed_at "$DEPLOYMENT_COMPLETED_AT" \
    --arg workflow_run_id "$WORKFLOW_RUN_ID" \
    --arg workflow_run_url "$WORKFLOW_RUN_URL" \
    --arg result "$result" \
    --arg failure_reason "$failure_reason" \
    --arg previous_known_good_commit "$PREVIOUS_KNOWN_GOOD_COMMIT" \
    --arg previous_known_good_completed_at "$PREVIOUS_KNOWN_GOOD_COMPLETED_AT" \
    --arg rollback_result "$ROLLBACK_RESULT" \
    --arg rollback_restored_commit "$ROLLBACK_RESTORED_COMMIT" \
    --argjson rollback_attempted "$rollback_attempted" \
    --argjson rollback_verified "$rollback_verified" \
    '
      def empty_to_null:
        if . == "" then null else . end;
      {
        record_type: $record_type,
        release_note: ($release_note | empty_to_null),
        requested_commit: ($requested_commit | empty_to_null),
        requested_at: ($requested_at | empty_to_null),
        started_at: ($started_at | empty_to_null),
        completed_at: ($completed_at | empty_to_null),
        workflow_run_id: ($workflow_run_id | empty_to_null),
        workflow_run_url: ($workflow_run_url | empty_to_null),
        result: ($result | empty_to_null),
        failure_reason: ($failure_reason | empty_to_null),
        previous_known_good_commit: ($previous_known_good_commit | empty_to_null),
        previous_known_good_completed_at: ($previous_known_good_completed_at | empty_to_null),
        rollback_attempted: $rollback_attempted,
        rollback_verified: $rollback_verified,
        rollback_result: ($rollback_result | empty_to_null),
        rollback_restored_commit: ($rollback_restored_commit | empty_to_null)
      }
    ' >"$tmp_file"

  install -o root -g root -m 0644 "$tmp_file" "$destination"
  rm -f "$tmp_file"
}

write_last_attempt_record() {
  local result="$1"
  local failure_reason="${2:-}"

  write_release_record "$LAST_ATTEMPT_RECORD" "last_attempt" "$result" "$failure_reason"
}

write_last_known_good_record() {
  local tmp_file

  tmp_file="$(mktemp)"
  jq -n \
    --arg commit "$REQUESTED_COMMIT" \
    --arg completed_at "$DEPLOYMENT_COMPLETED_AT" \
    --arg release_note "$RELEASE_NOTE" \
    --arg workflow_run_id "$WORKFLOW_RUN_ID" \
    --arg workflow_run_url "$WORKFLOW_RUN_URL" \
    '
      def empty_to_null:
        if . == "" then null else . end;
      {
        record_type: "last_known_good",
        commit: ($commit | empty_to_null),
        completed_at: ($completed_at | empty_to_null),
        release_note: ($release_note | empty_to_null),
        workflow_run_id: ($workflow_run_id | empty_to_null),
        workflow_run_url: ($workflow_run_url | empty_to_null),
        result: "success"
      }
    ' >"$tmp_file"

  install -o root -g root -m 0644 "$tmp_file" "$LAST_KNOWN_GOOD_RECORD"
  rm -f "$tmp_file"
}

safe_copy_if_exists() {
  local source="$1"
  local destination="$2"

  [[ -e "$source" ]] || return 0
  mkdir -p "$(dirname "$destination")"
  cp -a "$source" "$destination"
}

capture_http_evidence() {
  local name="$1"
  local url="$2"
  local output_file="$EVIDENCE_DIR/$name.txt"
  local exit_code=0
  local errexit_was_on=0

  if [[ $- == *e* ]]; then
    errexit_was_on=1
  fi

  set +e
  {
    printf 'GET %s\n\n' "$url"
    curl -i -sS --max-time 15 "$url"
    exit_code=$?
    printf '\n[curl_exit_code]=%s\n' "$exit_code"
  } >"$output_file" 2>&1
  if [[ "$errexit_was_on" -eq 1 ]]; then
    set -e
  fi
}

capture_command_evidence() {
  local name="$1"
  local output_file="$EVIDENCE_DIR/$name.txt"
  local exit_code=0
  local errexit_was_on=0
  shift

  if [[ $- == *e* ]]; then
    errexit_was_on=1
  fi

  set +e
  {
    printf '$'
    printf ' %q' "$@"
    printf '\n'
    "$@"
    exit_code=$?
    printf '\n[exit_code]=%s\n' "$exit_code"
  } >"$output_file" 2>&1
  if [[ "$errexit_was_on" -eq 1 ]]; then
    set -e
  fi
}

write_evidence_summary() {
  local stage="$1"
  local result="$2"
  local failure_reason="${3:-}"
  local summary_file="$EVIDENCE_DIR/summary.md"

  cat >"$summary_file" <<EOF
# Finance FaaS Deploy Evidence

- Stage: ${stage}
- Result: ${result}
- Release note: ${RELEASE_NOTE}
- Requested commit: ${REQUESTED_COMMIT:-<none>}
- Requested at: ${DEPLOYMENT_REQUESTED_AT:-<unknown>}
- Started at: ${DEPLOYMENT_STARTED_AT:-<unknown>}
- Completed at: ${DEPLOYMENT_COMPLETED_AT:-<unknown>}
- Workflow run id: ${WORKFLOW_RUN_ID:-<none>}
- Workflow run url: ${WORKFLOW_RUN_URL:-<none>}
- Previous known good commit: ${PREVIOUS_KNOWN_GOOD_COMMIT:-<none>}
- Previous known good completed at: ${PREVIOUS_KNOWN_GOOD_COMPLETED_AT:-<none>}
- Rollback attempted: $(bool_json "$ROLLBACK_ATTEMPTED")
- Rollback verified: $(bool_json "$ROLLBACK_VERIFIED")
- Rollback result: ${ROLLBACK_RESULT}
- Rollback restored commit: ${ROLLBACK_RESTORED_COMMIT:-<none>}
- Failure reason: ${failure_reason:-<none>}
EOF
}

capture_deploy_evidence() {
  local stage="$1"
  local result="$2"
  local failure_reason="${3:-}"
  local company_encoded_url

  mkdir -p "$EVIDENCE_DIR"

  write_release_record "$EVIDENCE_DIR/metadata.json" "deploy_evidence" "$result" "$failure_reason"
  safe_copy_if_exists "$LAST_ATTEMPT_RECORD" "$EVIDENCE_DIR/last_attempt.json"
  safe_copy_if_exists "$LAST_KNOWN_GOOD_RECORD" "$EVIDENCE_DIR/last_known_good.json"
  safe_copy_if_exists "$DEPLOYMENT_AUDIT_LOG" "$EVIDENCE_DIR/deploy-history.jsonl"

  capture_http_evidence "http-system-deployment" "$(control_api_url "/system/deployment")"
  capture_http_evidence "http-system-write-lock" "$(control_api_url "/system/write-lock")"
  capture_http_evidence "http-state" "$(control_api_url "/state")"
  capture_http_evidence "http-automation-health" "http://$CONTROL_HOST:50001/health"
  capture_http_evidence "http-automation-ready" "http://$CONTROL_HOST:50001/ready"
  capture_http_evidence "http-documentation-health" "http://$CONTROL_HOST:50101/health"
  capture_http_evidence "http-documentation-ready" "http://$CONTROL_HOST:50101/ready"
  capture_http_evidence "http-documentation-typedoc-status" "http://$CONTROL_HOST:50101/typedoc-status"
  capture_http_evidence "http-documentation-root" "http://$CONTROL_HOST:50101/"
  capture_http_evidence "http-capetown-health" "http://$CONTROL_HOST:50102/health"
  capture_http_evidence "http-db-health" "http://$CONTROL_HOST:50103/health"
  capture_http_evidence "http-companies" "http://$CONTROL_HOST:50102/companies"
  capture_http_evidence "http-public-dbcore-boundary" "http://$CONTROL_HOST/dbcore/"

  company_encoded_url="http://$CONTROL_HOST:50103/debtors/configuration?company=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$COMPANY")"
  capture_http_evidence "http-debtors-configuration" "$company_encoded_url"

  capture_command_evidence "systemctl-is-active" \
    "$SYSTEMCTL_BIN" is-active automation-core.service documentation-core.service capetown-faas-core.service database-abstraction-core.service nginx.service
  capture_command_evidence "systemctl-status-automation-core" "$SYSTEMCTL_BIN" --no-pager --full status automation-core.service
  capture_command_evidence "systemctl-status-documentation-core" "$SYSTEMCTL_BIN" --no-pager --full status documentation-core.service
  capture_command_evidence "systemctl-status-capetown-faas-core" "$SYSTEMCTL_BIN" --no-pager --full status capetown-faas-core.service
  capture_command_evidence "systemctl-status-database-abstraction-core" "$SYSTEMCTL_BIN" --no-pager --full status database-abstraction-core.service
  capture_command_evidence "systemctl-status-nginx" "$SYSTEMCTL_BIN" --no-pager --full status nginx.service

  write_evidence_summary "$stage" "$result" "$failure_reason"
  chmod -R a+rX "$EVIDENCE_DIR" || true
}

render_unit() {
  local template="$1"
  local destination="$2"
  local repo_root_escaped env_file_escaped tmp

  [[ -f "$template" ]] || fail "Missing systemd template: $template"

  repo_root_escaped="$(escape_sed_replacement "$REPO_ROOT")"
  env_file_escaped="$(escape_sed_replacement "$ENV_FILE")"
  tmp="$(mktemp)"

  sed \
    -e "s|<repo-root>|$repo_root_escaped|g" \
    -e "s|<env-file>|$env_file_escaped|g" \
    "$template" >"$tmp"

  install -o root -g root -m 0644 "$tmp" "$destination"
  rm -f "$tmp"
}

install_systemd_units() {
  local template_dir="$REPO_ROOT/infra/systemd"
  local unit
  for unit in automation-core.service documentation-core.service capetown-faas-core.service database-abstraction-core.service; do
    render_unit "$template_dir/$unit" "/etc/systemd/system/$unit"
  done
}

install_nginx_site() {
  local template destination enabled_link repo_root_escaped tmp

  template="$REPO_ROOT/infra/nginx/$NGINX_SITE_NAME.conf"
  [[ -f "$template" ]] || fail "Missing nginx template: $template"

  destination="/etc/nginx/sites-available/$NGINX_SITE_NAME"
  enabled_link="/etc/nginx/sites-enabled/$NGINX_SITE_NAME"
  repo_root_escaped="$(escape_sed_replacement "$REPO_ROOT")"
  tmp="$(mktemp)"

  sed -e "s|<repo-root>|$repo_root_escaped|g" "$template" >"$tmp"
  install -o root -g root -m 0644 "$tmp" "$destination"
  rm -f "$tmp"

  ln -sfn "$destination" "$enabled_link"
}

install_nginx_error_page() {
  local source_page destination_page

  source_page="$REPO_ROOT/apps/portal-web/dist/500.html"
  destination_page="$NGINX_ERROR_PAGE_ROOT/500.html"
  if [[ ! -f "$source_page" ]]; then
    warn "Skipping nginx error-page refresh because the source page is missing: $source_page"
    return 1
  fi

  mkdir -p "$NGINX_ERROR_PAGE_ROOT"
  install -o root -g root -m 0644 "$source_page" "$destination_page"
}

preserve_runtime_state() {
  [[ -d "$REPO_ROOT" ]] || fail "Repo root does not exist: $REPO_ROOT"
  [[ -f "$ARTIFACT" ]] || fail "Artifact not found: $ARTIFACT"

  WORK_DIR="$(mktemp -d /tmp/finance-faas-deploy.XXXXXX)"
  BACKUP_ROOT="$WORK_DIR/current"

  update_deployment_state "maintenance" "" "" "true" \
    "Safe deploy boundary reached. Read-only browsing may be briefly unavailable while services restart."
  install_nginx_error_page || true
  log "Stopping managed services before deploy"
  stop_service automation-core.service
  stop_service capetown-faas-core.service
  stop_service database-abstraction-core.service
  stop_service documentation-core.service

  log "Parking current runtime tree at $BACKUP_ROOT"
  mv "$REPO_ROOT" "$BACKUP_ROOT"
  mkdir -p "$REPO_ROOT"
  NEEDS_ROLLBACK=1
}

restore_backup_tree() {
  local failed_root=""
  if [[ "$NEEDS_ROLLBACK" -ne 1 ]]; then
    return 0
  fi
  if [[ ! -d "$BACKUP_ROOT" ]]; then
    return 0
  fi

  log "Restoring original runtime tree"
  failed_root="$WORK_DIR/failed-runtime"
  rm -rf "$failed_root"
  if [[ -e "$REPO_ROOT" ]]; then
    log "Moving failed runtime tree aside to $failed_root"
    mv "$REPO_ROOT" "$failed_root" || {
      warn "Unable to move the failed runtime tree aside from $REPO_ROOT"
      return 1
    }
  fi
  if [[ -e "$REPO_ROOT" ]]; then
    warn "Runtime root still exists after moving the failed tree aside: $REPO_ROOT"
    return 1
  fi
  mv "$BACKUP_ROOT" "$REPO_ROOT" || {
    warn "Unable to restore the backup runtime tree into $REPO_ROOT"
    return 1
  }
  ensure_runtime_directories || {
    warn "Unable to recreate the runtime directory structure after rollback"
    return 1
  }
  chown -R "$SERVICE_USER:$SERVICE_USER" "$REPO_ROOT" || true
  install_systemd_units || true
  install_nginx_site || true
  install_nginx_error_page || true
  "$SYSTEMCTL_BIN" daemon-reload || true
  "$SYSTEMCTL_BIN" start database-abstraction-core.service || true
  "$SYSTEMCTL_BIN" start documentation-core.service || true
  "$SYSTEMCTL_BIN" start capetown-faas-core.service || true
  "$SYSTEMCTL_BIN" start automation-core.service || true
  maybe_reload_nginx || true
  NEEDS_ROLLBACK=0
}

cleanup_temp() {
  if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then
    rm -rf "$WORK_DIR"
  fi
}

ensure_runtime_directories() {
  mkdir -p \
    "$REPO_ROOT/logs" \
    "$REPO_ROOT/logs/services" \
    "$REPO_ROOT/services/capetown-faas-core/.faas-state" \
    "$REPO_ROOT/services/database-abstraction-core/.faas-state"
}

sync_database_company_map_seeds() {
  local source_dir="$REPO_ROOT/services/database-abstraction-core/config/company-maps"
  local target_dir="$REPO_ROOT/services/database-abstraction-core/.faas-state"
  local file_name

  [[ -d "$source_dir" ]] || fail "Missing database company-map seed directory: $source_dir"
  mkdir -p "$target_dir"

  for file_name in \
    stats-company-map.json \
    cashier-company-map.json \
    creditors-company-map.json \
    icapv-company-map.json \
    automatedmail-company-map.json; do
    [[ -f "$source_dir/$file_name" ]] || fail "Missing database company-map seed file: $source_dir/$file_name"
    cp "$source_dir/$file_name" "$target_dir/$file_name"
  done
}

verify_public_dbcore_boundary() {
  local path status

  for path in /dbcore /dbcore/; do
    status="$(curl -sS -o /dev/null -w "%{http_code}" "http://$CONTROL_HOST$path" || true)"
    if [[ "$status" != "404" ]]; then
      fail "Public nginx /dbcore boundary is not closed; expected HTTP 404 from http://$CONTROL_HOST$path, got ${status:-<no-status>}"
    fi
  done
}

perform_smoke_checks() {
  local portal_root_html

  curl -fsS "http://$CONTROL_HOST:50001/health" | jq -e '.service == "automation-core" and .status == "ok"' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50001/ready" | jq -e '.ready == true' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50101/health" | jq -e '.service == "documentation-core" and .status == "ok"' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50101/ready" | jq -e '.ready == true and (.build_error == null)' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50101/typedoc-status" | jq -e '.service == "documentation-core" and .mode == "typedoc" and .ready == true and (.build_error == null)' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50101/" >/dev/null
  curl -fsS "http://$CONTROL_HOST:50102/health" | jq -e '.service == "capetown-faas-core" and .status == "ok"' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50103/health" | jq -e '.service == "database-abstraction-core" and .status == "ok"' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50102/companies" | jq -e --arg company "$COMPANY" '.companies | index($company) != null' >/dev/null
  curl -fsS "http://$CONTROL_HOST:50103/debtors/configuration?company=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$COMPANY")" | jq -e '.configured == true and .live_ready == true and .configuration_source == "statistics_aux_db"' >/dev/null
  portal_root_html="$(curl -fsS "http://$CONTROL_HOST/")"
  grep -F 'id="docs-link"' <<<"$portal_root_html" >/dev/null
  grep -F 'id="forget-me-button"' <<<"$portal_root_html" >/dev/null
  grep -F 'id="inspector-toggle"' <<<"$portal_root_html" >/dev/null
  grep -F 'id="theme-toggle"' <<<"$portal_root_html" >/dev/null
  curl -fsS "http://$CONTROL_HOST/500.html" >/dev/null
  curl -fsS "http://$CONTROL_HOST/debtors/" >/dev/null
  verify_public_dbcore_boundary
}

verify_rollback_runtime() {
  log "Verifying restored runtime after rollback"
  wait_for_json "http://$CONTROL_HOST:50101/health" '.service == "documentation-core" and .status == "ok"' 60 2
  wait_for_json "http://$CONTROL_HOST:50101/ready" '.ready == true and (.build_error == null)' 120 2
  wait_for_json "http://$CONTROL_HOST:50103/health" '.service == "database-abstraction-core" and .status == "ok"' 60 2
  wait_for_json "http://$CONTROL_HOST:50102/health" '.service == "capetown-faas-core" and .status == "ok"' 60 2
  wait_for_json "http://$CONTROL_HOST:50001/health" '.service == "automation-core" and .status == "ok"' 60 2
  wait_for_json "http://$CONTROL_HOST:50001/ready" '.ready == true' 60 2
  perform_smoke_checks
}

rollback_and_verify() {
  ROLLBACK_ATTEMPTED=1
  if ! restore_backup_tree; then
    ROLLBACK_RESULT="restore_failed"
    return 1
  fi

  ROLLBACK_RESTORED_COMMIT="$PREVIOUS_KNOWN_GOOD_COMMIT"
  if verify_rollback_runtime; then
    ROLLBACK_VERIFIED=1
    ROLLBACK_RESULT="verified"
    return 0
  fi

  ROLLBACK_RESULT="verification_failed"
  return 1
}

on_error() {
  local line="$1"
  local failure_reason deployment_result maintenance_pending

  trap - ERR
  set +e
  log "Deployment failed at line $line"

  failure_reason="Deployment failed at line $line"
  deployment_result="failed"
  maintenance_pending="true"
  local status_detail="Deployment failed during maintenance. Writes remain paused for operator review."

  if [[ "$DEPLOYMENT_VERIFIED" -ne 1 && "$DEPLOYMENT_MUTATION_STARTED" -eq 1 ]]; then
    if rollback_and_verify; then
      deployment_result="rolled_back"
      maintenance_pending="false"
      status_detail="Previous runtime restored successfully. Writes reopened automatically after rollback verification."
      log "Rollback restored the previous runtime and passed smoke verification"
      disable_system_write_lock "Rollback verified after failed deployment: ${RELEASE_NOTE}" || true
    else
      deployment_result="rollback_failed"
      maintenance_pending="true"
      status_detail="Rollback could not be verified. Write lock remains enabled for operator review."
      log "Rollback could not be verified; keeping write lock enabled for operator review"
    fi
  elif [[ "$WRITE_LOCK_CHANGED_BY_SCRIPT" -eq 1 && "$DEPLOYMENT_MUTATION_STARTED" -eq 0 ]]; then
    maintenance_pending="false"
    status_detail="Deployment aborted before runtime mutation. Writes reopened automatically."
    log "Clearing system write lock because deployment aborted before runtime mutation"
    disable_system_write_lock "Deployment aborted before runtime mutation: ${RELEASE_NOTE}" || true
  elif [[ "$WRITE_LOCK_CHANGED_BY_SCRIPT" -eq 1 ]]; then
    log "System write lock remains enabled for operator review after deployment failure"
  fi

  DEPLOYMENT_COMPLETED_AT="$(timestamp_now)"
  update_deployment_state "failed" "$deployment_result" "$failure_reason" "$maintenance_pending" "$status_detail"
  write_last_attempt_record "$deployment_result" "$failure_reason" || true
  capture_deploy_evidence "failed" "$deployment_result" "$failure_reason" || true
  log "Temporary files preserved at: ${WORK_DIR:-<none>}"
  exit 1
}

abort_deployment_without_runtime_mutation() {
  local failure_reason="$1"
  local deployment_result="${2:-failed}"
  local status_detail="${3:-Deployment aborted before runtime mutation. Writes reopened automatically.}"

  trap - ERR
  set +e
  log "$failure_reason"
  disable_system_write_lock "Deployment aborted before runtime mutation: ${RELEASE_NOTE}" || true
  DEPLOYMENT_COMPLETED_AT="$(timestamp_now)"
  update_deployment_state "failed" "$deployment_result" "$failure_reason" "false" "$status_detail" "true" "false"
  write_last_attempt_record "$deployment_result" "$failure_reason" || true
  capture_deploy_evidence "failed" "$deployment_result" "$failure_reason" || true
  cleanup_temp
  exit 1
}

extract_artifact() {
  local extract_dir="$WORK_DIR/extract"
  local source_root
  local top_level_entries=()

  mkdir -p "$extract_dir"
  tar -xf "$ARTIFACT" -C "$extract_dir"

  mapfile -t top_level_entries < <(find "$extract_dir" -mindepth 1 -maxdepth 1 -printf '%P\n' | sort)
  if [[ "${#top_level_entries[@]}" -eq 1 && -d "$extract_dir/${top_level_entries[0]}" ]]; then
    source_root="$extract_dir/${top_level_entries[0]}"
  else
    source_root="$extract_dir"
  fi

  log "Syncing artifact into $REPO_ROOT"
  update_deployment_state "deploying" "" "" "true" \
    "Applying the release artifact, refreshing runtime environments, and restarting managed services."
  cp -a "$source_root"/. "$REPO_ROOT"/
}

restore_state_artifacts() {
  local service

  if [[ -f "$BACKUP_ROOT/.env" ]]; then
    cp -a "$BACKUP_ROOT/.env" "$REPO_ROOT/.env"
  fi

  if [[ -d "$BACKUP_ROOT/logs" ]]; then
    mkdir -p "$REPO_ROOT/logs"
    cp -a "$BACKUP_ROOT/logs"/. "$REPO_ROOT/logs"/
  fi

  for service in automation-core documentation-core capetown-faas-core database-abstraction-core; do
    if [[ -d "$BACKUP_ROOT/services/$service/.faas-state" ]]; then
      mkdir -p "$REPO_ROOT/services/$service"
      cp -a "$BACKUP_ROOT/services/$service/.faas-state" "$REPO_ROOT/services/$service/"
    fi
  done

  if [[ "$SKIP_PORTAL_BUILD" -eq 1 && -d "$BACKUP_ROOT/apps/portal-web/dist" ]]; then
    mkdir -p "$REPO_ROOT/apps/portal-web"
    cp -a "$BACKUP_ROOT/apps/portal-web/dist" "$REPO_ROOT/apps/portal-web/"
  fi
}

fix_repo_ownership() {
  chown -R "$SERVICE_USER:$SERVICE_USER" "$REPO_ROOT"
}

sync_python_envs() {
  local service_dir
  for service_dir in \
    "$REPO_ROOT/services/automation-core" \
    "$REPO_ROOT/services/documentation-core" \
    "$REPO_ROOT/services/capetown-faas-core" \
    "$REPO_ROOT/services/database-abstraction-core"; do
    log "Refreshing uv environment in ${service_dir#$REPO_ROOT/}"
    run_as_service_user "cd '$service_dir' && '$UV_BIN' sync --python '$PYTHON_VERSION'"
  done
}

append_optional_cli_arg() {
  local flag="$1"
  local value="${2:-}"
  if [[ -z "$value" ]]; then
    return 0
  fi
  printf ' %q %q' "$flag" "$value"
}

run_repo_node_command() {
  local command_text="$1"
  run_as_service_user "
    cd '$REPO_ROOT'
    export NVM_DIR='/home/$SERVICE_USER/.nvm'
    if [[ -s \"\$NVM_DIR/nvm.sh\" ]]; then
      . \"\$NVM_DIR/nvm.sh\"
      nvm use --silent >/dev/null
    fi
    $command_text
  "
}

compute_portal_source_fingerprint() {
  run_repo_node_command "node scripts/write_portal_build_provenance.mjs --print-fingerprint"
}

read_portal_provenance_field() {
  local field_name="$1"
  local provenance_path="$REPO_ROOT/apps/portal-web/dist/build-provenance.json"

  [[ -f "$provenance_path" ]] || return 1
  jq -r --arg field_name "$field_name" '.[$field_name] // empty' "$provenance_path"
}

write_portal_build_provenance() {
  local reused_flag="${1:-false}"
  local built_at_utc="${2:-}"
  local provenance_args=""

  provenance_args+="$(append_optional_cli_arg --commit "$REQUESTED_COMMIT")"
  provenance_args+="$(append_optional_cli_arg --deployed-commit "$REQUESTED_COMMIT")"
  provenance_args+="$(append_optional_cli_arg --workflow-run-id "$WORKFLOW_RUN_ID")"
  provenance_args+="$(append_optional_cli_arg --reused "$reused_flag")"
  provenance_args+="$(append_optional_cli_arg --built-at "$built_at_utc")"

  run_repo_node_command "node scripts/write_portal_build_provenance.mjs$provenance_args"
}

install_root_node_tools() {
  log "Refreshing root Node tooling for the documentation build"
  run_repo_node_command "
    '$NPM_BIN' --version >/dev/null
    if [[ -f package-lock.json ]]; then
      '$NPM_BIN' ci
    else
      '$NPM_BIN' install
    fi
  "
}

build_documentation_site() {
  log "Building the TypeDoc documentation site"
  run_as_service_user "
    cd '$REPO_ROOT'
    export NVM_DIR='/home/$SERVICE_USER/.nvm'
    if [[ -s \"\$NVM_DIR/nvm.sh\" ]]; then
      . \"\$NVM_DIR/nvm.sh\"
      nvm use --silent >/dev/null
    fi
    '$NPM_BIN' run docs:build
  "
}

build_portal_web() {
  local requested_source_fingerprint=""
  local preserved_source_fingerprint=""
  local preserved_built_at_utc=""

  if [[ "$SKIP_PORTAL_BUILD" -eq 1 ]]; then
    requested_source_fingerprint="$(compute_portal_source_fingerprint)"
    preserved_source_fingerprint="$(read_portal_provenance_field "sourceFingerprint" || true)"
    preserved_built_at_utc="$(read_portal_provenance_field "builtAtUtc" || true)"

    if [[ -z "$preserved_source_fingerprint" ]]; then
      warn "Portal build skip was requested, but no preserved dist provenance was found; rebuilding portal-web."
    elif [[ "$preserved_source_fingerprint" != "$requested_source_fingerprint" ]]; then
      warn "Portal build skip was requested, but preserved dist fingerprint did not match the requested portal source fingerprint; rebuilding portal-web."
    else
      log "Skipping portal-web build because preserved dist fingerprint matches the requested portal source fingerprint"
      write_portal_build_provenance "true" "$preserved_built_at_utc"
      install_nginx_error_page
      return 0
    fi
  fi

  log "Refreshing portal-web node dependencies and build"
  run_repo_node_command "
    '$NPM_BIN' --version >/dev/null
    if [[ -f apps/portal-web/package-lock.json ]]; then
      '$NPM_BIN' --prefix apps/portal-web ci
    else
      '$NPM_BIN' --prefix apps/portal-web install
    fi
    '$NPM_BIN' --prefix apps/portal-web run build
  "
  write_portal_build_provenance "false"
  install_nginx_error_page
}

restart_services() {
  "$SYSTEMCTL_BIN" daemon-reload
  "$SYSTEMCTL_BIN" enable automation-core.service documentation-core.service capetown-faas-core.service database-abstraction-core.service

  start_service documentation-core.service
  wait_for_json "http://$CONTROL_HOST:50101/health" '.service == "documentation-core" and .status == "ok"' 60 2
  wait_for_json "http://$CONTROL_HOST:50101/ready" '.ready == true and (.build_error == null)' 120 2
  wait_for_json "http://$CONTROL_HOST:50101/typedoc-status" '.service == "documentation-core" and .mode == "typedoc" and .ready == true and (.build_error == null)' 120 2
  wait_for_http "http://$CONTROL_HOST:50101/" 60 2

  start_service database-abstraction-core.service
  wait_for_json "http://$CONTROL_HOST:50103/health" '.service == "database-abstraction-core" and .status == "ok"' 60 2

  start_service capetown-faas-core.service
  wait_for_json "http://$CONTROL_HOST:50102/health" '.service == "capetown-faas-core" and .status == "ok"' 60 2

  start_service automation-core.service
  wait_for_json "http://$CONTROL_HOST:50001/health" '.service == "automation-core" and .status == "ok"' 60 2
  wait_for_json "http://$CONTROL_HOST:50001/ready" '.ready == true' 60 2
}

run_smoke_checks() {
  log "Running smoke checks"
  update_deployment_state "verifying" "" "" "true" \
    "Running smoke checks before writes and new logins resume."
  perform_smoke_checks
}

maybe_reload_nginx() {
  if command -v "$NGINX_BIN" >/dev/null 2>&1; then
    "$NGINX_BIN" -t
    "$SYSTEMCTL_BIN" reload nginx
  fi
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --artifact)
      ARTIFACT="${2:-}"
      shift 2
      ;;
    --repo-root)
      REPO_ROOT="${2:-}"
      shift 2
      ;;
    --env-file)
      ENV_FILE="${2:-}"
      shift 2
      ;;
    --control-host)
      CONTROL_HOST="${2:-}"
      shift 2
      ;;
    --company)
      COMPANY="${2:-}"
      shift 2
      ;;
    --requested-at)
      DEPLOYMENT_REQUESTED_AT="${2:-}"
      shift 2
      ;;
    --release-note)
      RELEASE_NOTE="${2:-}"
      shift 2
      ;;
    --requested-commit)
      REQUESTED_COMMIT="${2:-}"
      shift 2
      ;;
    --workflow-run-id)
      WORKFLOW_RUN_ID="${2:-}"
      shift 2
      ;;
    --workflow-run-url)
      WORKFLOW_RUN_URL="${2:-}"
      shift 2
      ;;
    --service-user)
      SERVICE_USER="${2:-}"
      shift 2
      ;;
    --python-version)
      PYTHON_VERSION="${2:-}"
      shift 2
      ;;
    --uv-bin)
      UV_BIN="${2:-}"
      shift 2
      ;;
    --npm-bin)
      NPM_BIN="${2:-}"
      shift 2
      ;;
    --systemctl)
      SYSTEMCTL_BIN="${2:-}"
      shift 2
      ;;
    --nginx-bin)
      NGINX_BIN="${2:-}"
      shift 2
      ;;
    --lock-file)
      LOCK_FILE="${2:-}"
      shift 2
      ;;
    --release-state-root)
      RELEASE_STATE_ROOT="${2:-}"
      shift 2
      ;;
    --evidence-dir)
      EVIDENCE_DIR="${2:-}"
      shift 2
      ;;
    --skip-portal-build)
      SKIP_PORTAL_BUILD=1
      shift
      ;;
    --write-lock-mode)
      WRITE_LOCK_MODE="${2:-}"
      shift 2
      ;;
    --write-lock-requester)
      WRITE_LOCK_REQUESTER="${2:-}"
      shift 2
      ;;
    --drain-timeout)
      DRAIN_TIMEOUT_SECONDS="${2:-}"
      shift 2
      ;;
    --drain-interval)
      DRAIN_INTERVAL_SECONDS="${2:-}"
      shift 2
      ;;
    --max-active-sessions)
      MAX_ACTIVE_SESSIONS="${2:-}"
      shift 2
      ;;
    --active-session-grace-seconds)
      ACTIVE_SESSION_GRACE_SECONDS="${2:-}"
      shift 2
      ;;
    --close-active-sessions)
      CLOSE_ACTIVE_SESSIONS="$(parse_bool_flag "${2:-}")"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      fail "Unknown argument: $1"
      ;;
  esac
done

trap 'on_error $LINENO' ERR
trap release_deploy_lock EXIT

[[ $EUID -eq 0 ]] || fail "This script must be run as root."
require_command tar
require_command cp
require_command mv
require_command sed
require_command find
require_command curl
require_command jq
require_command python3
require_command flock
require_executable "$UV_BIN"
require_executable "$SYSTEMCTL_BIN"

case "$WRITE_LOCK_MODE" in
  off|drain)
    ;;
  *)
    fail "Unsupported --write-lock-mode value: $WRITE_LOCK_MODE"
    ;;
esac

[[ -n "$ARTIFACT" ]] || fail "Missing --artifact path."
[[ -f "$ARTIFACT" ]] || fail "Artifact does not exist: $ARTIFACT"
[[ -d "$REPO_ROOT" ]] || fail "Repo root does not exist: $REPO_ROOT"
[[ -f "$ENV_FILE" ]] || fail "EnvironmentFile does not exist: $ENV_FILE"

ensure_release_state_paths
log "Starting deployment (${RELEASE_NOTE})"
append_deploy_audit_record "deploy_started" "normal" "" "" "false" \
  "Deployment script started and is waiting to acquire the deploy lock." "true" "false"
acquire_deploy_lock
enable_system_write_lock
close_active_sessions_for_deploy
if ! wait_for_safe_deploy_boundary; then
  abort_deployment_without_runtime_mutation \
    "Timed out waiting for a safe deploy boundary after ${DRAIN_TIMEOUT_SECONDS}s." \
    "drain_timeout" \
    "Drain timeout reached before the system became idle enough for maintenance."
fi
DEPLOYMENT_MUTATION_STARTED=1
preserve_runtime_state
extract_artifact
restore_state_artifacts
ensure_runtime_directories
sync_database_company_map_seeds
fix_repo_ownership
install_systemd_units
install_nginx_site
sync_python_envs
install_root_node_tools
build_documentation_site
build_portal_web
restart_services
maybe_reload_nginx
run_smoke_checks
DEPLOYMENT_VERIFIED=1
DEPLOYMENT_COMPLETED_AT="$(timestamp_now)"
write_last_attempt_record "success" ""
write_last_known_good_record
disable_system_write_lock "Deployment completed successfully: ${RELEASE_NOTE}"
update_deployment_state "resumed" "success" "" "false"
capture_deploy_evidence "success" "success" "" || true

NEEDS_ROLLBACK=0
cleanup_temp
log "Deployment completed successfully (${RELEASE_NOTE})."