#!/usr/bin/env bash
# qalizer-run.sh — Qalizer CI local Docker runner bootstrap.
# Exit: 0 passed, 1 test fail, 2 infra/usage, 3 auth, 4 timeout/cancelled
#
# CONTRACT (published with the script; the Profile generators and ci/tests use
# exactly this table)
# ---------------------------------------------------------------------------
#   --monitor ID              required
#   --scenario ID             one of --scenario / --scenario-set / --scenario-ids
#   --scenario-set NAME
#   --scenario-ids "1,2,3"
#   --location LOCATION       auto | local_only | server_only — sent as
#                             execution_location and required to be honoured
#   --token-file PATH         read the CI token from a protected file
#   --env-file PATH           extra environment for the runner container
#   --protected-file PATH     a credential file mounted READ-ONLY at the same
#                             path — never in argv, env, labels or logs
#   --api-base URL  --image IMG  --image-tag TAG  --output-dir DIR
#   --timeout S  --poll-interval S  --connect-timeout S  --request-timeout S
#   --keep-image  --remove-image  --no-cleanup  --help
#
# Secrets are NEVER required on the command line: the token comes from
# QALIZER_CI_TOKEN or --token-file, and every credential file this script
# creates or is given is removed on exit — INCLUDING under --no-cleanup, which
# only keeps the (already stopped) CONTAINER.
set -euo pipefail

API_BASE="${QALIZER_API_BASE:-https://qalizer.online}"
VERSION_URL="${QALIZER_VERSION_URL:-/api/v1/ci/runner-version}"
RUNNER_IMAGE="${QALIZER_RUNNER_IMAGE:-}"
RUNNER_IMAGE_TAG="${QALIZER_RUNNER_IMAGE_TAG:-}"
POLL_INTERVAL="${QALIZER_POLL_INTERVAL:-5}"
TIMEOUT_SECONDS="${QALIZER_TIMEOUT_SECONDS:-1800}"
OUTPUT_DIR="${QALIZER_OUTPUT_DIR:-./qalizer-reports}"
ENABLE_CLEANUP="${QALIZER_CLEANUP:-true}"
KEEP_IMAGE="${QALIZER_KEEP_IMAGE:-false}"
REMOVE_IMAGE="${QALIZER_REMOVE_IMAGE:-false}"
# Every NETWORK call is bounded: a connect timeout, a total per-request
# timeout, and an overall limit for the whole operation.
CONNECT_TIMEOUT="${QALIZER_CONNECT_TIMEOUT:-10}"
REQUEST_TIMEOUT="${QALIZER_REQUEST_TIMEOUT:-60}"
TOTAL_LIMIT="${QALIZER_TOTAL_LIMIT:-$((TIMEOUT_SECONDS + 900))}"

MONITOR_ID=""; SCENARIO_ID=""; SCENARIO_SET=""; SCENARIO_IDS=""; TOKEN=""
LOCATION="${QALIZER_LOCATION:-auto}"
TOKEN_FILE_IN=""; ENV_FILE_IN=""; PROTECTED_FILES=()
AUTH_FILE=""; CONTAINER_ENV_FILE=""; CONTAINER_ID=""; DOCKER_STARTED="false"
JOB_ID=""; SUBMIT_STATUS=""; SUBMITTED="false"; HTTP_STATUS=""

say()  { printf '[qalizer] %s\n' "$*"; }
warn() { printf '[qalizer] WARN: %s\n' "$*" >&2; }
fail_with() { local code="$1"; shift; printf '[qalizer] ERROR: %s\n' "$*" >&2; exit "$code"; }

# qcurl reports the HTTP status as the final QALIZERHTTP:<status> token. Callers need the BODY *and* the status, and
# ``var="$(cmd)"`` runs in a SUBSHELL where a variable set inside is lost — so
# the status travels WITH the body and ``qbody`` splits them apart.
HTTP_STATUS_SENTINEL="QALIZERHTTP:"

usage() {
  cat <<'EOF'
Usage:
  qalizer-run.sh --monitor ID --scenario ID --location auto|local_only|server_only
  qalizer-run.sh --monitor ID --scenario-set NAME --location ...
  qalizer-run.sh --monitor ID --scenario-ids "101,102" --location ...

  The CI token is read from QALIZER_CI_TOKEN (preferred) or --token-file PATH.
  --token TOKEN is accepted for compatibility only: it is visible in the
  process list and is NOT recommended.

Options: --monitor --scenario --scenario-set --scenario-ids --location
  --token --token-file --env-file --protected-file
  --api-base --image --image-tag --output-dir --timeout --poll-interval
  --connect-timeout --request-timeout --keep-image --remove-image
  --no-cleanup --help

Exit codes: 0 passed, 1 test failure, 2 infrastructure/usage,
            3 authentication/authorization, 4 timeout/cancelled.
EOF
}

while [ $# -gt 0 ]; do
  case "$1" in
    --monitor)          MONITOR_ID="${2:-}"; shift 2 ;;
    --scenario)         SCENARIO_ID="${2:-}"; shift 2 ;;
    --scenario-set)     SCENARIO_SET="${2:-}"; shift 2 ;;
    --scenario-ids)     SCENARIO_IDS="${2:-}"; shift 2 ;;
    --token|--token-file|--env-file|--protected-file|--location|--api-base|\
    --image|--image-tag|--output-dir|--timeout|--poll-interval|\
    --connect-timeout|--request-timeout)
      # A VALUE OPTION must actually carry a value: without this check a missing
      # argument silently consumed the NEXT option (e.g. ``--protected-file``
      # with no path ate ``--output-dir DIR``, and the report directory was then
      # treated as a credential file to delete).
      case "${2:-}" in
        ""|-*) fail_with 2 "Option $1 requires a value" ;;
      esac
      case "$1" in
        --token)           TOKEN="$2" ;;
        --token-file)      TOKEN_FILE_IN="$2" ;;
        --env-file)        ENV_FILE_IN="$2" ;;
        --protected-file)  PROTECTED_FILES+=("$2") ;;
        --location)        LOCATION="$2" ;;
        --api-base)        API_BASE="$2" ;;
        --image)           RUNNER_IMAGE="$2" ;;
        --image-tag)       RUNNER_IMAGE_TAG="$2" ;;
        --output-dir)      OUTPUT_DIR="$2" ;;
        --timeout)         TIMEOUT_SECONDS="$2" ;;
        --poll-interval)   POLL_INTERVAL="$2" ;;
        --connect-timeout) CONNECT_TIMEOUT="$2" ;;
        --request-timeout) REQUEST_TIMEOUT="$2" ;;
      esac
      shift 2 ;;
    --keep-image)       KEEP_IMAGE="true"; shift ;;
    --remove-image)     REMOVE_IMAGE="true"; shift ;;
    --no-cleanup)       ENABLE_CLEANUP="false"; shift ;;
    --help|-h)          usage; exit 0 ;;
    *) fail_with 2 "Unknown argument: $1" ;;
  esac
done

# ── Token: environment > protected file > legacy argument ────────────────────
if [ -z "$TOKEN" ] && [ -n "$TOKEN_FILE_IN" ]; then
  [ -r "$TOKEN_FILE_IN" ] || fail_with 3 "The token file is not readable: ${TOKEN_FILE_IN}"
  TOKEN="$(tr -d '\r\n' < "$TOKEN_FILE_IN")"
fi
if [ -z "$TOKEN" ] && [ -n "${QALIZER_CI_TOKEN:-}" ]; then
  TOKEN="$QALIZER_CI_TOKEN"
fi
[ -n "$TOKEN" ]      || fail_with 2 "Missing CI token: set QALIZER_CI_TOKEN or pass --token-file"
[ -n "$MONITOR_ID" ] || fail_with 2 "Missing required argument: --monitor"
sel=0; [ -n "$SCENARIO_ID" ] && sel=$((sel+1)); [ -n "$SCENARIO_SET" ] && sel=$((sel+1)); [ -n "$SCENARIO_IDS" ] && sel=$((sel+1))
[ "$sel" -eq 1 ] || fail_with 2 "Provide exactly one of --scenario, --scenario-set, --scenario-ids"

# ── Execution location: an explicit, VALIDATED value ─────────────────────────
# An unsupported value is refused HERE, never rewritten to ``auto``: a typo in
# --location must not silently run the tests somewhere else.
case "$LOCATION" in
  auto|local_only|server_only) : ;;
  *) fail_with 2 "Unsupported --location '${LOCATION}' (expected auto, local_only or server_only)" ;;
esac

printf '%s' "$TOKEN" | grep -q '[^A-Za-z0-9_\-]' && fail_with 3 "Invalid token"
printf '%s' "$MONITOR_ID" | grep -Eq '^[A-Za-z0-9\-]+$' || fail_with 2 "Invalid monitor id"
[ -z "$SCENARIO_ID" ] || printf '%s' "$SCENARIO_ID" | grep -Eq '^[0-9]+$' || fail_with 2 "Invalid scenario id"
[ -z "$SCENARIO_IDS" ] || printf '%s' "$SCENARIO_IDS" | grep -Eq '^[0-9]+(,[0-9]+)*$' || fail_with 2 "Invalid scenario-ids"
[ -z "$SCENARIO_SET" ] || printf '%s' "$SCENARIO_SET" | grep -Eq '^[A-Za-z0-9 _.\\-]+$' || fail_with 2 "Invalid scenario-set"

case "$API_BASE" in
  https://*) : ;;
  http://*) [ "${QALIZER_ALLOW_HTTP:-false}" = "true" ] || fail_with 2 "HTTPS only" ;;
  *) fail_with 2 "Invalid API base" ;;
esac


# --- Bounded network calls --------------------------------------------------------------------
# EVERY network call goes through ``qcurl``: it bounds the connect and the whole
# request and never lets a response body decide the HTTP status.
#
# WHY the body is base64-packed: a captured body must survive ``var="$(...)"``
# byte-for-byte, and the HTTP status must be readable WITHOUT parsing the body
# (a response containing the status token must not be able to forge one).
# ``qcurl`` therefore prints exactly ONE line:
#
#     <base64 of the body> <3-digit status>
#
# ``qstatus`` reads the last field, ``qbody`` decodes the first one. A body can
# never forge a status: the status is a separate, fixed-width FIELD, not text
# the server controls.
qcurl() {  # <curl args...>  -- one line: "<base64 body> <status>"; curl's rc
  local out rc started
  out="$(mktemp "${TMPDIR:-/tmp}/qalizer-curl-out-XXXXXX")"
  started="$(date +%s)"
  set +e
  curl --config "$AUTH_FILE" \
    --connect-timeout "$CONNECT_TIMEOUT" --max-time "$REQUEST_TIMEOUT" \
    --silent --show-error \
    --output "$out" --write-out "%{http_code}" \
    "$@" > "${out}.status"
  rc=$?
  set -e
  # ``--max-time`` bounds a STALLED transfer; a trickling one is bounded here.
  if [ $(( $(date +%s) - started )) -gt "$REQUEST_TIMEOUT" ]; then rc=28; fi
  HTTP_STATUS_CAPTURE="$(tr -d '\r\n' < "${out}.status" 2>/dev/null)"
  [ -n "$HTTP_STATUS_CAPTURE" ] || HTTP_STATUS_CAPTURE="000"
  if [ -s "$out" ]; then
    printf '%s %s\n' "$(base64 -w 0 < "$out")" "$HTTP_STATUS_CAPTURE"
  else
    printf '%s %s\n' "" "$HTTP_STATUS_CAPTURE"
  fi
  rm -f "$out" "${out}.status"
  return $rc
}

# The HTTP status of a captured qcurl output (its last field), or "" when the
# capture did not carry one.
qstatus() {  # <raw>
  local raw
  raw="$(printf '%s' "${1:-}" | tr -d '\n')"
  case "$raw" in
    *" "[0-9][0-9][0-9]) printf '%s' "${raw##* }" ;;
    *) printf '' ;;
  esac
}

# The BODY of a captured qcurl output (the first field, decoded).
qbody() {  # <raw>
  local raw b64
  raw="$(printf '%s' "${1:-}" | tr -d '\n')"
  b64="${raw% *}"
  [ -n "$b64" ] || return 0
  printf '%s' "$b64" | base64 -d
}
# Distinguish the API's answers by their ACTUAL HTTP status.
report_auth_rejected() {  # <what> <status>
  local what="$1" status="$2" msg=""
  case "$status" in
    401) SUBMIT_STATUS="unauthorized"; msg="the CI token was rejected (HTTP 401)" ;;
    403) SUBMIT_STATUS="forbidden";    msg="the CI token is not allowed to do this (HTTP 403)" ;;
    402) SUBMIT_STATUS="usage_limit";  msg="the account cannot run this now: no usable credits or an unpaid plan (HTTP 402)" ;;
    *)   SUBMIT_STATUS="authentication_failed"; msg="authentication/authorization failed (HTTP ${status:-unknown})" ;;
  esac
  say "Qalizer API ${what} failed: ${msg}"
  exit 3
}
report_api_failure() {  # <what> <status>
  local what="$1" status="$2"
  case "$status" in
    401|402|403) report_auth_rejected "$what" "$status" ;;
    422)
      say "Qalizer API ${what} was refused as INVALID (HTTP 422). The request, not the" >&2
      say "runner, is wrong — nothing was executed." >&2
      fail_with 2 "Invalid request" ;;
    429)
      say "Qalizer API ${what} was rate limited (HTTP 429). Nothing was executed;" >&2
      say "retry later." >&2
      fail_with 2 "Rate limited by the Qalizer API" ;;
    5??)
      say "Qalizer API ${what} failed on the Qalizer server (HTTP ${status})." >&2
      fail_with 2 "Qalizer API server error" ;;
    ""|000)
      say "Qalizer API ${what} could not be reached (network error or timeout)." >&2
      fail_with 2 "Qalizer API unreachable" ;;
    *)
      say "Qalizer API ${what} failed with an unexpected HTTP status ${status}." >&2
      fail_with 2 "Unexpected Qalizer API response" ;;
  esac
}


command -v jq >/dev/null 2>&1 || fail_with 2 "jq required"
command -v curl >/dev/null 2>&1 || fail_with 2 "curl required"
command -v docker >/dev/null 2>&1 || fail_with 2 "Docker not installed"
docker info >/dev/null 2>&1 || fail_with 2 "Docker not running"
say "Docker is available"
say "Execution location requested: ${LOCATION}"

umask 077
TMP_ROOT="${TMPDIR:-/tmp}"
AUTH_FILE="$(mktemp "${TMP_ROOT}/qalizer-curl-XXXXXX.conf")"
printf 'header = "Authorization: Bearer %s"\n' "$TOKEN" > "$AUTH_FILE"

CONTAINER_ENV_FILE="$(mktemp "${TMP_ROOT}/qalizer-ci-XXXXXX.env")"
{
  printf 'QALIZER_CI_TOKEN=%s\n' "$TOKEN"
  printf 'QALIZER_JOB_ID=unknown\n'
  printf 'QALIZER_API_URL=%s\n' "$API_BASE"
} > "$CONTAINER_ENV_FILE"
if [ -n "$ENV_FILE_IN" ]; then
  [ -r "$ENV_FILE_IN" ] || fail_with 2 "The --env-file is not readable: ${ENV_FILE_IN}"
  printf '\n' >> "$CONTAINER_ENV_FILE"
  cat "$ENV_FILE_IN" >> "$CONTAINER_ENV_FILE"
fi
# Credential files to destroy on exit: the two this script created PLUS every
# file the caller explicitly supplied (a supplied token file holds the secret).
MASKED_FILES=("$AUTH_FILE" "$CONTAINER_ENV_FILE")
if [ -n "$TOKEN_FILE_IN" ]; then MASKED_FILES+=("$TOKEN_FILE_IN"); fi
# Partial artifact transfers (``*.part.<pid>``) are tracked so a crash mid
# download can never leave a truncated file that LOOKS like an artifact.
PART_FILES=()

# Credential files are MOUNTED READ-ONLY at the same absolute path inside the
# container. They are never passed with -e/--env, never written into a label or
# the container name, and never echoed: a CI log must not contain them.
DOCKER_MOUNT_ARGS=()
for _protected in ${PROTECTED_FILES[@]+"${PROTECTED_FILES[@]}"}; do
  [ -n "$_protected" ] || continue
  [ -f "$_protected" ] || fail_with 2 "The --protected-file does not exist: ${_protected}"
  case "$_protected" in
    /*) : ;;
    *) fail_with 2 "The --protected-file must be an absolute path: ${_protected}" ;;
  esac
  DOCKER_MOUNT_ARGS+=("--mount" "type=bind,source=${_protected},target=${_protected},readonly")
  MASKED_FILES+=("$_protected")
done

# The trap ALWAYS removes the credential files this script created, and every
# credential file it was handed, EVEN under --no-cleanup (which only keeps the
# stopped container). A crash mid-run therefore never leaves a token on disk.
cleanup() {
  local rc=$?
  set +e
  if [ -n "$CONTAINER_ID" ] && [ "$DOCKER_STARTED" = "true" ]; then
    if [ "$ENABLE_CLEANUP" = "true" ]; then
      docker rm -f "$CONTAINER_ID" >/dev/null 2>&1
      warn "Removed temporary container $CONTAINER_ID"
    else
      docker stop "$CONTAINER_ID" >/dev/null 2>&1 \
        || warn "Could not stop container $CONTAINER_ID (--no-cleanup)"
      warn "Container kept by --no-cleanup: ${CONTAINER_ID}"
    fi
  fi
  # Only the two files THIS script created (random suffix below TMPDIR) and the
  # credential files the caller EXPLICITLY supplied with --token-file /
  # --protected-file are ever removed here — never an output-dir, a report or
  # any other path that was merely mentioned (or accidentally parsed) as an
  # argument.
  local _file
  for _file in ${MASKED_FILES[@]+"${MASKED_FILES[@]}"}; do
    [ -n "$_file" ] || continue
    [ -f "$_file" ] || continue
    if ! rm -f -- "$_file" 2>/dev/null; then
      warn "Could not remove credential file: ${_file}"
    fi
  done
  # Partial artifact transfers are removed unconditionally: a truncated body is
  # never allowed to survive as a file that could be mistaken for a report.
  for _file in ${PART_FILES[@]+"${PART_FILES[@]}"}; do
    [ -n "$_file" ] || continue
    rm -f -- "$_file" 2>/dev/null || true
  done
  exit "$rc"
}
trap cleanup EXIT
trap 'exit 4' INT TERM

# Fail fast when a utility the script depends on is missing (a missing
# ``mkdir`` would otherwise turn into an endless, unbounded run).
for _need in mktemp mkdir sed grep tr base64 date sleep; do
  command -v "$_need" >/dev/null 2>&1     || fail_with 2 "Required utility not found on PATH: ${_need}"
done

# The WHOLE operation is bounded from HERE — before the image pull, before any
# download and before polling — so a missing utility or a hung docker command
# can never leave the run unbounded.
OPERATION_STARTED="$(date +%s)"
# ONE bound for the WHOLE operation. The helper installs no traps and never
# signals itself (bash cannot run a subshell's trap while it waits for a child,
# which leaked a process); it expires and interrupts the MAIN shell through the
# INT handler, which reports exit code 4.
run_watchdog() {
  while :; do
    sleep 5
    if [ $(( $(date +%s) - OPERATION_STARTED )) -ge "$TOTAL_LIMIT" ]; then
      printf '[qalizer] ERROR: the whole operation exceeded %ss - stopping\n' \
        "$TOTAL_LIMIT" >&2
      kill -TERM "$WATCHDOG_TARGET" 2>/dev/null || true
      return 0
    fi
  done
}
WATCHDOG_TARGET="$$"
run_watchdog &
WATCHDOG_PID=$!

# ── Runner version configuration (bounded, status-aware) ────────────────────
say "Fetching runner version configuration from ${API_BASE}${VERSION_URL}"
# The assignment's exit status IS curl's, and the status is split out of the
# captured output by ``qbody`` (a subshell cannot export a variable).
version_raw="$(qcurl "${API_BASE}${VERSION_URL}")" || true
version_json="$(qbody "$version_raw")"
HTTP_STATUS="$(qstatus "$version_raw")"
case "$HTTP_STATUS" in
  2??) : ;;
  *) report_api_failure "runner-version" "$HTTP_STATUS" ;;
esac
if ! printf '%s' "$version_json" | jq -e . >/dev/null 2>&1; then
  # The version endpoint is PUBLIC, so a non-JSON body means the API is not
  # answering from the expected service (a proxy error page, a captive portal).
  fail_with 2 "The runner-version endpoint did not return JSON; refusing to guess the image."
fi

[ -n "$RUNNER_IMAGE" ] || RUNNER_IMAGE="$(printf '%s' "$version_json" | jq -r '.image // empty')"
[ -n "$RUNNER_IMAGE" ] || RUNNER_IMAGE="denis20/qalizer-runner"
[ -n "$RUNNER_IMAGE_TAG" ] || RUNNER_IMAGE_TAG="$(printf '%s' "$version_json" | jq -r '.tag // empty')"
[ -n "$RUNNER_IMAGE_TAG" ] || RUNNER_IMAGE_TAG="1.0.1"

case "$RUNNER_IMAGE_TAG" in
  sha256:*) IMAGE_REF="${RUNNER_IMAGE}@${RUNNER_IMAGE_TAG}" ;;
  *)        IMAGE_REF="${RUNNER_IMAGE}:${RUNNER_IMAGE_TAG}" ;;
esac
say "Using pinned runner image: ${IMAGE_REF}"

# ── Verified entrypoint ──────────────────────────────────────────────────────
# The version endpoint publishes the entrypoint that EXISTS inside the selected
# image. The container is started with exactly that path (never a guessed one), and
# an unverified entrypoint is refused BEFORE the run is submitted — otherwise the
# container dies at boot and the job looks like a mysterious test failure.
RUNNER_ENTRYPOINT="$(printf '%s' "$version_json" | jq -r '.entrypoint // ""')"
ENTRYPOINT_VERIFIED="$(printf '%s' "$version_json" | jq -r '.entrypoint_verified // false')"
if [ -n "$RUNNER_ENTRYPOINT" ]; then
  case "$ENTRYPOINT_VERIFIED" in
    true) : ;;
    *) fail_with 2 "The runner image ${IMAGE_REF} publishes an UNVERIFIED entrypoint (${RUNNER_ENTRYPOINT}). Refusing to start a container that cannot boot." ;;
  esac
  case "$RUNNER_ENTRYPOINT" in
    /*) : ;;
    *) fail_with 2 "Unsafe runner entrypoint published by the API: ${RUNNER_ENTRYPOINT}" ;;
  esac
fi
if [ -n "$RUNNER_ENTRYPOINT" ]; then
  say "Runner entrypoint (verified in ${IMAGE_REF}): ${RUNNER_ENTRYPOINT}"
else
  say "Runner image ${IMAGE_REF} defines its own ENTRYPOINT (none overridden)"
fi

if [ "$RUNNER_IMAGE_TAG" = "latest" ] && [ "${QALIZER_ALLOW_LATEST:-false}" != "true" ]; then
  fail_with 2 "Refusing unpinned runner image tag 'latest'. Pin a version or SHA-256 digest."
fi

# Create CI run payload (argument-safe, shell-injection-free via jq)
scenario_ids_json="[]"
if [ -n "$SCENARIO_IDS" ]; then
  scenario_ids_json="$(printf '%s' "$SCENARIO_IDS" | tr ',' '\n' | jq -R . | jq -s .)"
fi

payload="$(jq -cn \
  --arg monitor "$MONITOR_ID" \
  --arg scenario "$SCENARIO_ID" \
  --arg scenario_set "$SCENARIO_SET" \
  --arg location "$LOCATION" \
  --argjson scenario_ids "$scenario_ids_json" \
  '{monitor_id:$monitor,
    scenario_id:(if $scenario=="" then null else ($scenario|tonumber) end),
    scenario_set_name:(if $scenario_set=="" then null else $scenario_set end),
    scenario_ids:$scenario_ids,
    execution_location:$location,
    source:"ci",
    environment:"production"}')"

say "Submitting CI run to ${API_BASE}/api/v1/ci/runs (execution_location=${LOCATION}) ..."
submit_raw="$(qcurl -H 'Content-Type: application/json' \
  --data-binary "$payload" \
  "${API_BASE}/api/v1/ci/runs")" || true
submit_resp="$(qbody "$submit_raw")"
HTTP_STATUS="$(qstatus "$submit_raw")"
case "$HTTP_STATUS" in
  2??) : ;;
  *) report_api_failure "create-run" "$HTTP_STATUS" ;;
esac

JOB_ID="$(printf '%s' "$submit_resp" | jq -r '.job_id // empty')"
if [ -z "$JOB_ID" ]; then
  fail_with 2 "CI API did not return a job_id. Response: $(printf '%s' "$submit_resp" | head -c 300)"
fi
say "CI job created: ${JOB_ID}"
sed -i "s/^QALIZER_JOB_ID=.*/QALIZER_JOB_ID=${JOB_ID}/" "$CONTAINER_ENV_FILE"


# Pull pinned runner image — bounded by the same watchdog.
say "Pulling runner image ${IMAGE_REF} (this may take a minute on first run)..."
if ! docker pull "$IMAGE_REF"; then
  fail_with 2 "Failed to pull runner image: ${IMAGE_REF}"
fi

# Run Docker image with temporary storage. The token is supplied via a mounted
# read-only protected env file — never via -e, never in the container name,
# never in labels, never in argv.
CONTAINER_ID="qalizer-ci-run-${CI_PIPELINE_ID:-$(date +%s)}-$$-${RANDOM}"
# The token is supplied ONLY through the mounted read-only env file: never `-e`,
# never argv, never URL, never logs. The entrypoint is the VERIFIED one published
# for this image (a compiled image defines its own, so nothing is overridden).
DOCKER_ENTRYPOINT_ARGS=()
if [ -n "$RUNNER_ENTRYPOINT" ]; then
  DOCKER_ENTRYPOINT_ARGS=(--entrypoint /bin/bash "$IMAGE_REF"
    -c 'set -a; . /run/qalizer/ci.env 2>/dev/null || true; set +a; exec "$1"' _ "$RUNNER_ENTRYPOINT")
else
  DOCKER_ENTRYPOINT_ARGS=("$IMAGE_REF")
fi
say "Starting runner container ${CONTAINER_ID} (temporary storage enabled)..."
if ! docker run -d --rm \
  --name "$CONTAINER_ID" \
  --mount "type=bind,source=${CONTAINER_ENV_FILE},target=/run/qalizer/ci.env,readonly" \
  ${DOCKER_MOUNT_ARGS[@]+"${DOCKER_MOUNT_ARGS[@]}"} \
  --tmpfs /tmp:rw,noexec,nosuid,size=1g \
  "${DOCKER_ENTRYPOINT_ARGS[@]}" >/dev/null 2>&1; then
  CONTAINER_ID=""
  fail_with 2 "Failed to start the runner container from ${IMAGE_REF}"
fi
DOCKER_STARTED="true"
say "Runner container started"

# Poll CI status endpoint
STATUS_URL="${API_BASE}/api/v1/ci/runs/${JOB_ID}"
start_epoch="$(date +%s)"
deadline=$(( start_epoch + TIMEOUT_SECONDS ))
FINAL_STATUS=""
FINAL_ERROR_CODE=""
say "Polling ${STATUS_URL}"

while :; do
  status_raw="$(qcurl "$STATUS_URL")" || true
  status_json="$(qbody "$status_raw")"
  HTTP_STATUS="$(qstatus "$status_raw")"
  case "$HTTP_STATUS" in
    2??) : ;;
    000)
      # A transient network blip is retried until the deadline.
      warn "CI status request failed (network); retrying"
      sleep "$POLL_INTERVAL"
      continue ;;
    *) report_api_failure "run-status" "$HTTP_STATUS" ;;
  esac

  status="$(printf '%s' "$status_json" | jq -r '.status // empty')"
  passed="$(printf '%s' "$status_json" | jq -r '.passed // 0')"
  failed="$(printf '%s' "$status_json" | jq -r '.failed // 0')"
  total="$(printf '%s' "$status_json" | jq -r '.total // 0')"

  # Absolute operation bound: never act on a run past TOTAL_LIMIT, whatever the
  # run's own timeout says.
  if [ $(( $(date +%s) - OPERATION_STARTED )) -ge "$TOTAL_LIMIT" ]; then
    say "The operation exceeded ${TOTAL_LIMIT}s - giving up (the run is cancelled)"
    qcurl -X POST "${API_BASE}/api/v1/ci/runs/${JOB_ID}/cancel" >/dev/null 2>&1 || true
    FINAL_STATUS="timed_out"
    FINAL_ERROR_CODE="timeout"
    break
  fi

  case "$status" in
    passed)
      say "CI run ${JOB_ID} PASSED (${passed}/${total} passed)"
      FINAL_STATUS="passed"; break ;;
    failed)
      say "CI run ${JOB_ID} FAILED (${passed}/${total} passed, ${failed} failed)"
      FINAL_STATUS="failed"; break ;;
    error)
      err_msg="$(printf '%s' "$status_json" | jq -r '.error_message // empty')"
      err_code="$(printf '%s' "$status_json" | jq -r '.error_code // empty')"
      say "CI run ${JOB_ID} ERROR: ${err_msg:-unknown execution error} (code: ${err_code:-unknown})"
      FINAL_STATUS="error"
      FINAL_ERROR_CODE="$err_code"
      break ;;
    cancelled)
      say "CI run ${JOB_ID} CANCELLED"
      FINAL_STATUS="cancelled"; break ;;
    timed_out)
      say "CI run ${JOB_ID} TIMED OUT"
      FINAL_STATUS="timed_out"; break ;;
    queued|running|""|null)
      now_epoch="$(date +%s)"
      elapsed=$(( now_epoch - start_epoch ))
      if [ "$now_epoch" -ge "$deadline" ]; then
        say "CI run ${JOB_ID} not finished within ${TIMEOUT_SECONDS}s — cancelling it"
        qcurl -X POST "${API_BASE}/api/v1/ci/runs/${JOB_ID}/cancel" >/dev/null 2>&1 || true
        FINAL_STATUS="timed_out"
        FINAL_ERROR_CODE="timeout"
        # The cancellation of a RUNNING browser execution may be refused; the
        # timeout is still reported truthfully, and the job stays owned by the
        # backend (no unaccounted execution is created HERE).
        break
      fi
      printf '[qalizer] waiting... status=%s elapsed=%ss (%.0f%% of %ss)\r' \
        "${status:-queued}" "$elapsed" "$((elapsed * 100 / TIMEOUT_SECONDS))" "$TIMEOUT_SECONDS"
      sleep "$POLL_INTERVAL"
      ;;
    *)
      # An UNKNOWN status is never treated as success and never ignored: the
      # operation is bounded, so it fails instead of polling forever.
      warn "Unexpected status from the CI API: '${status}'"
      now_epoch="$(date +%s)"
      if [ "$now_epoch" -ge "$deadline" ]; then
        FINAL_STATUS="error"
        FINAL_ERROR_CODE="unexpected_status"
        break
      fi
      sleep "$POLL_INTERVAL"
      ;;
  esac
done
printf '\n'

# ── Artifacts ────────────────────────────────────────────────────────────────
# The artifacts are DOWNLOADED here (before any upload step) and are the evidence
# the exit code is based on: the JUnit document must itself prove a green run.
mkdir -p "$OUTPUT_DIR"
JUNIT_LOCAL=""
JUNIT_HTTP=""
# Downloads ONE artifact format BINARY-SAFELY. Returns ``"<status> <path>"``
# (path empty when nothing usable was downloaded) so the CALLER can classify the
# ACTUAL status.
#
# WHY this does NOT go through ``qcurl``: ``qcurl`` base64-packs the body so the
# status can travel with it through ``var="$(...)"`` — but the *download* is the
# one place a body must be preserved EXACTLY. A shell variable loses trailing
# newlines (``$(...)`` strips them) and cannot hold a NUL byte at all, so a PDF
# or any binary artifact was silently corrupted by ``printf '%s' "$(qbody ...)"``.
#
# The contract here is therefore:
#   * curl writes the body STRAIGHT to a temporary file next to the artifact;
#   * the HTTP status travels on its OWN stream (``--write-out``), never through
#     the body, so a response can never forge one;
#   * curl's exit code is captured SEPARATELY: a 2xx whose transfer did not
#     complete (partial body, timeout, receive error) is NOT an artifact;
#   * the temporary file is promoted by an ATOMIC rename ONLY on a complete,
#     successful 2xx transfer; every other outcome removes it.
download_report() {  # fmt ext
  local fmt="$1" ext="$2"
  local final="${OUTPUT_DIR}/qalizer-${JOB_ID}-report.${ext}"
  local tmp="${final}.part.$$"
  local status rc
  rm -f "$final" "$tmp"
  PART_FILES+=("$tmp")
  # ``set +e`` around the call: the assignment's exit status IS curl's, and a
  # transport failure must be CLASSIFIED here, not abort the whole script.
  set +e
  status="$(curl --config "$AUTH_FILE" \
    --connect-timeout "$CONNECT_TIMEOUT" --max-time "$REQUEST_TIMEOUT" \
    --silent --show-error \
    --output "$tmp" --write-out '%{http_code}' \
    "${API_BASE}/api/v1/ci/runs/${JOB_ID}/report/download?format=${fmt}" \
    2>/dev/null)"
  rc=$?
  set -e
  status="$(printf '%s' "$status" | tr -d '\r\n')"
  [ -n "$status" ] || status="000"
  case "$status" in
    2??)
      if [ "$rc" -ne 0 ]; then
        # 2xx header, INCOMPLETE body (curl rc 18/28/56/…). The bytes on disk are
        # a truncated transfer and are NEVER promoted to a completed artifact.
        rm -f "$tmp"
        warn "Report format ${fmt}: the response was HTTP ${status} but the transfer did not complete (curl rc=${rc}). The partial file was removed; it is not an artifact."
        printf 'transfer_failed\n'
      else
        mv -f "$tmp" "$final"
        printf '200 %s\n' "$final"
      fi ;;
    *)
      rm -f "$tmp"
      printf '%s\n' "$status" ;;
  esac
}

for _fmt in json html pdf junit; do
  # Register the partial path in the PARENT shell too: the function runs inside a
  # command substitution (its own subshell), so a registration made there would
  # never reach the exit trap. The naming is deterministic (``$$`` is inherited
  # by the subshell), so the trap can always find and remove a partial.
  PART_FILES+=("${OUTPUT_DIR}/qalizer-${JOB_ID}-report.${_fmt}.part.$$")
  read -r _status _path <<<"$(download_report "$_fmt" "$_fmt")"
  case "$_status" in
    2??)
      say "Downloaded ${_fmt} report: ${_path}"
      if [ "$_fmt" = "junit" ]; then JUNIT_LOCAL="$_path"; JUNIT_HTTP="$_status"; fi ;;
    401|402|403)
      # The evidence cannot be read with this token: an authorization failure.
      report_auth_rejected "report/${_fmt}" "$_status" ;;
    transfer_failed)
      # A 2xx header with an incomplete body: the partial file was removed and
      # NOTHING was produced. It is an evidence/transport failure, never a pass —
      # and it is never described as an HTTP status the server did not send.
      if [ "$_fmt" = "junit" ]; then JUNIT_LOCAL=""; JUNIT_HTTP="transfer_failed"; fi
      warn "Report format ${_fmt}: the transfer was INCOMPLETE (truncated or timed out); no artifact was produced" ;;
    *)
      if [ "$_fmt" = "junit" ]; then JUNIT_LOCAL=""; JUNIT_HTTP="$_status"; fi
      warn "Report format ${_fmt} not available for this run (HTTP ${_status})" ;;
  esac
done

# ── Verify the ARTIFACT, not just the status attribute ──────────────────────
# ``junit_green`` is true ONLY when the downloaded JUnit document:
#   * parses as XML with exactly one <testsuite> root,
#   * carries coherent counts (tests/failures/errors/skipped are integers and
#     failures+errors+skipped <= tests),
#   * contains ACTUALLY EXECUTED cases (tests > 0 and at least one <testcase>),
#   * contains no <failure>, <error> or <skipped> element.
# An empty <testsuite/>, a skipped-only suite, an absent or unparseable file is
# NOT green. The check FAILS CLOSED when no XML parser is available: a grep over
# XML text is not XML validation and is never used as a pass.
junit_green() {
  [ -n "$JUNIT_LOCAL" ] && [ -s "$JUNIT_LOCAL" ] || return 1
  command -v python3 >/dev/null 2>&1 || {
    warn "No XML parser (python3) is available: refusing to certify the JUnit artifact."
    return 1
  }
  python3 - "$JUNIT_LOCAL" <<'PY'
import sys
try:
    import xml.etree.ElementTree as ET
except Exception:
    sys.exit(1)          # no parser ⇒ fail closed


def to_int(value, default=-1):
    try:
        return int(str(value).strip())
    except Exception:
        return default


try:
    root = ET.parse(sys.argv[1]).getroot()
except Exception:
    sys.exit(1)          # unparseable ⇒ not green

if root.tag != "testsuite":
    sys.exit(1)          # one supported root element only

cases = root.findall("testcase")
declared = to_int(root.get("tests"), -1)
failures = to_int(root.get("failures"), 0)
errors = to_int(root.get("errors"), 0)
skipped_attr = to_int(root.get("skipped"), 0)
for name, value in (("tests", declared), ("failures", failures),
                    ("errors", errors), ("skipped", skipped_attr)):
    if value < 0:
        sys.exit(1)      # incoherent counts are not evidence

if declared <= 0 or len(cases) == 0:
    sys.exit(1)          # nothing was executed ⇒ nothing is proven
if declared != len(cases):
    sys.exit(1)          # the counts disagree with the body

if failures or errors or skipped_attr:
    sys.exit(1)
for case in cases:
    if case.find("failure") is not None or case.find("error") is not None \
            or case.find("skipped") is not None:
        sys.exit(1)      # unproven / failed / skipped-only results never pass
sys.exit(0)
PY
}

# Stable exit codes: 0 passed, 1 test fail, 2 infra/error, 3 auth, 4 timeout/cancelled
case "$FINAL_STATUS" in
  passed)
    if junit_green; then
      exit_code=0; say "Qalizer CI run ${JOB_ID} PASSED (JUnit artifact verified)"
    else
      # The server said passed but the artifact does not prove it: the artifact is
      # authoritative (a run must never be green on an unproven artifact). A 404
      # means no report exists (nothing executed/proven); any other answer means
      # the evidence could not be read. Both are infrastructure failures.
      exit_code=2
      say "Qalizer CI run ${JOB_ID} reported passed, but its JUnit artifact does not" >&2
      if [ "${JUNIT_HTTP:-}" = "404" ]; then
        say "exist (HTTP 404: no report was produced for this run). Treated as an" >&2
        say "infrastructure/evidence failure, never as a pass." >&2
      elif [ "${JUNIT_HTTP:-}" = "transfer_failed" ]; then
        say "exist (the download was HTTP 2xx but the transfer did not COMPLETE — the" >&2
        say "partial file was removed). Treated as an infrastructure/evidence failure," >&2
        say "never as a pass." >&2
      else
        say "prove a green run (absent, unparseable, empty, skipped-only or containing" >&2
        say "failures; HTTP ${JUNIT_HTTP:-none}). Treated as an infrastructure/evidence" >&2
        say "failure, never as a pass." >&2
      fi
    fi
    ;;
  failed)
    exit_code=1; say "Qalizer CI run ${JOB_ID} FAILED (see reports in ${OUTPUT_DIR})" ;;
  error)
    # Map backend error codes → exit codes (not every failure is exit 1):
    #   unauthorized / forbidden                 → 3 (auth/authorization)
    #   timeout / cancelled                      → 4 (timeout/cancellation)
    #   empty_result / incomplete_execution      → 2 (nothing was proven)
    #   cleanup_unconfirmed                      → 2 (unsafe/unproven)
    #   everything else                          → 2 (runner/infrastructure)
    case "$FINAL_ERROR_CODE" in
      unauthorized|forbidden|invalid_token|authentication_failed)
        exit_code=3; say "Qalizer CI run ${JOB_ID} AUTH/PERMISSION ERROR (see reports in ${OUTPUT_DIR})" ;;
      timeout|cancelled|timed_out)
        exit_code=4; say "Qalizer CI run ${JOB_ID} TIMED OUT / CANCELLED" ;;
      empty_result)
        exit_code=2
        say "Qalizer CI run ${JOB_ID} produced NO executed results — nothing could be" >&2
        say "proven to pass. Reported as an infrastructure failure, never as a pass." >&2 ;;
      incomplete_execution)
        exit_code=2
        say "Qalizer CI run ${JOB_ID} did NOT execute all planned work — the run is" >&2
        say "incomplete and is never reported as a pass." >&2 ;;
      cleanup_unconfirmed)
        exit_code=2
        say "Qalizer CI run ${JOB_ID} could not confirm resource cleanup — its outcome" >&2
        say "is not trusted as passed." >&2 ;;
      *)
        exit_code=2; say "Qalizer CI run ${JOB_ID} ERRORED (see reports in ${OUTPUT_DIR})" ;;
    esac
    ;;
  cancelled|timed_out)
    exit_code=4; say "Qalizer CI run ${JOB_ID} ${FINAL_STATUS} — no result is proven" ;;
  *)
    exit_code=2; say "Qalizer CI run ${JOB_ID} ended with an unknown final status; refusing to" >&2
    say "report it as a pass." >&2 ;;
esac

# Stop the background watchdog: the operation is over.
if [ -n "${WATCHDOG_PID:-}" ]; then
  kill "$WATCHDOG_PID" >/dev/null 2>&1 || true
  wait "$WATCHDOG_PID" 2>/dev/null || true
fi

# Image removal only when explicitly requested (never automatic)
if [ "$REMOVE_IMAGE" = "true" ]; then
  say "Removing runner image ${IMAGE_REF} (explicitly requested)"
  docker rmi "$IMAGE_REF" >/dev/null 2>&1 || warn "Could not remove image ${IMAGE_REF}"
elif [ "$KEEP_IMAGE" = "true" ]; then
  say "Keeping runner image ${IMAGE_REF} (explicitly requested)"
fi

exit "$exit_code"