From 044b6b5798e1ebb324a681632404b38c4d03f123 Mon Sep 17 00:00:00 2001 From: Steve Leonard Date: Thu, 17 Sep 2026 08:15:04 +0100 Subject: [PATCH 1/2] Add pools.sh autoscale: resize pools to their queued jobs Fixed pool counts either hold idle runners' memory or leave jobs queueing. autoscale grows opted-in pools to busy + queued (capped at max) and shrinks them to min after an idle timeout, behind the same busy check as stop. Closes #110 Co-Authored-By: Claude Opus 5 --- .gitignore | 2 + README.md | 60 ++++++++++++- autoscale.conf.example | 14 +++ pools.sh | 194 +++++++++++++++++++++++++++++++++++++++++ tests/pools_test.sh | 102 +++++++++++++++++++++- 5 files changed, 368 insertions(+), 4 deletions(-) create mode 100644 autoscale.conf.example diff --git a/.gitignore b/.gitignore index fc8ac64..f7dd419 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ _work/ pools.conf pools.conf.bak pools.conf.tmp +autoscale.conf +.autoscale-state windows-pools.conf windows/runners/ windows/toolcache/ diff --git a/README.md b/README.md index f84ece5..dcb031a 100644 --- a/README.md +++ b/README.md @@ -145,9 +145,11 @@ here, most of which cost somebody an afternoon to find. If you need something else, use something else: -- **Kubernetes, autoscaling, or more than one team** -> +- **Kubernetes, autoscaling across a cluster, or more than one team** -> [actions-runner-controller](https://github.com/actions/actions-runner-controller), - which is the supported answer and does all of that properly. + which is the supported answer and does all of that properly. Resizing + one host's pools to their queued jobs is covered: see + [Autoscaling a host's pools](#autoscaling-a-hosts-pools). - **A public repository** -> nothing here, and see the next section. This is not a limitation to work around; it is the one hard rule. - **macOS jobs** -> not covered by anything in this repo. @@ -212,6 +214,7 @@ the orchestration was not buying anything. ./pools.sh declare [count] # add a pool to pools.conf, starting nothing ./pools.sh sync [--dry-run] # rewrite pools.conf to match the pools running here ./pools.sh list [--json [...]] # every pool on this host, and what GitHub actually sees +./pools.sh autoscale [--once] [--dry-run] [--interval ] # resize autoscale.conf's pools to their queued jobs ``` `` is just the label in the project name (`gh-runner-`); it need @@ -297,9 +300,60 @@ login, which needs admin access to the repo -- a classic token with `repo`, or a fine-grained one with **Administration: read**. Without it, the busy checks refuse and `list --json` reports `"runners": null`. -`tests/pools_test.sh` exercises `start`/`stop`/`restart`/`restart-runner`/`scale`/`sync`/`list --json` against stub +`tests/pools_test.sh` exercises `start`/`stop`/`restart`/`restart-runner`/`scale`/`sync`/`list --json`/`autoscale` against stub `docker` and `gh` commands, so it needs neither a Docker daemon nor GitHub. +### Autoscaling a host's pools + +A fixed count is always wrong for somebody: too many runners holding memory +while nothing is queued, or too few while jobs wait behind each other. +`./pools.sh autoscale` resizes the pools listed in `autoscale.conf` to what is +actually queued: + +```bash +cp autoscale.conf.example autoscale.conf # gitignored; one " [idle_minutes]" line per pool +./pools.sh autoscale --once --dry-run # what it would do right now, changing nothing +./pools.sh autoscale # every 60s until stopped; --interval to change that +``` + +Every pass, for each listed pool: + +- **Demand** is the jobs queued in its repo -- in queued runs, and in runs + already in progress with jobs still waiting -- whose `runs-on` labels this + pool's runners carry, including this host's `RUNNER_HOST_LABEL`. A job + more than one `pools.conf` pool could take counts against the one with the + fewest labels, so a pool reserved by an extra label (`issue-worm`) doesn't + grow for the generic jobs its repo's plain CI pool is there for. +- **Growing**: to busy runners plus queued jobs, capped at ``, when that + is more containers than the pool has, and never below ``. Containers + still starting count as capacity, so a job that stays queued while its + runner boots doesn't grow the pool a second time. +- **Shrinking**: to `` once the pool has had no busy runner and no + queued job for `[idle_minutes]` (10 by default), and only after the same + busy check `stop` makes. The idle clock is kept in `.autoscale-state`, so + it survives from one pass to the next. A pool with any busy runner is not + shrunk at all, because `docker compose up --scale` can't be told which + containers to remove. +- A pool whose repo GitHub can't be asked about is left as it is. + +It resizes with the same `docker compose up --scale` as `scale`, and never +writes `pools.conf`: that file's count is still what `start` and `restart` +use, and the next pass corrects it. Each pass reads the runs, their jobs and +the runners of every autoscaled repo -- a few API calls per repo per +minute, well inside the 5,000 an hour a token gets. + +Two hosts serving the same repo each see the same queued job, so each may +add a runner for it; the spare one sits idle and goes again after +`[idle_minutes]`. Nothing here sizes memory or CPU yet: a pool still gets +its `pools.conf` `[mem]`/`[pids]` limits. + +Keep it running the way you keep anything else on the host running -- a +systemd unit, or at the least: + +```bash +nohup ./pools.sh autoscale >> autoscale.log 2>&1 & +``` + `[label]`, `[mem]` and `[pids]` are optional, trailing, and positional -- pass `-` for one you want to leave at its default so a later one still lands in the right slot. They let a pool carry an extra runner label and its own diff --git a/autoscale.conf.example b/autoscale.conf.example new file mode 100644 index 0000000..81f1a80 --- /dev/null +++ b/autoscale.conf.example @@ -0,0 +1,14 @@ +# The pools `./pools.sh autoscale` resizes. One line per pool: +# +# cp autoscale.conf.example autoscale.conf # gitignored, edit freely +# +# must also be declared in pools.conf, which still supplies the repo, +# label and limits; its count there is ignored while it's autoscaled. +# is kept warm even with nothing queued -- 1 saves the first job the +# 30-60s a runner takes to start and register. [idle_minutes] is how long a +# pool must have no busy runner and no queued job before it drops back to +# ; 10 if omitted. +# +# name min max idle_minutes +issue-worm-pro 1 8 +sing-attune 0 3 5 diff --git a/pools.sh b/pools.sh index b4f76a4..a59d593 100644 --- a/pools.sh +++ b/pools.sh @@ -32,6 +32,8 @@ Usage: ./pools.sh declare [count] add a pool to pools.conf, starting nothing ./pools.sh sync [--dry-run] rewrite pools.conf to match the pools here ./pools.sh list [--json [...]] every pool this host knows about + ./pools.sh autoscale [--once] [--dry-run] [--interval ] + resize autoscale.conf's pools to their queued jobs is the short label used in the project name, e.g. "jobtrack" for gh-runner-jobtrack. It does not have to match the repo name. @@ -107,6 +109,17 @@ repo are told apart; it is null when GitHub couldn't be asked. Names after --json limit it to those pools -- worth doing from anything that polls, since every pool costs a GitHub API call; a name that is neither declared nor running is left out. + +autoscale resizes the pools autoscale.conf lists (one " +[idle_minutes]" line each; every name must also be in pools.conf) every +--interval seconds, 60 by default, or once with --once. A pool grows to its +busy runners plus the queued jobs it could take, capped at , when that +is more containers than it has; it shrinks to once it has had no busy +runner and no queued job for [idle_minutes] (default 10), and only after the +same busy check stop makes. A queued job that several pools.conf pools could +take counts against the one with the fewest labels, so a pool reserved by an +extra label doesn't grow for jobs the plain pool serves. pools.conf is never +rewritten. --dry-run prints what it would do and changes nothing. EOF } @@ -609,6 +622,186 @@ cmd_list_json() { printf ']\n' } +AUTOSCALE_CONF="autoscale.conf" +# " " for each pool idle above its min, and since when, so the +# idle timeout survives from one pass to the next. +AUTOSCALE_STATE=".autoscale-state" + +# RUNNER_HOST_LABEL as compose resolves it: the environment, then .env. +host_label() { + local label="${RUNNER_HOST_LABEL:-}" + if [[ -z "$label" && -f .env ]]; then + label="$(tr -d '\r' < .env | sed -n 's/^RUNNER_HOST_LABEL=//p' | tail -1)" + fi + printf '%s\n' "${label:-unlabelled-host}" +} + +# Succeeds if comma-separated label list includes every label in . +labels_cover() { + local want l + IFS=, read -ra want <<< "$2" + for l in "${want[@]}"; do + [[ ",$1," == *",$l,"* ]] || return 1 + done +} + +# One lowercased, comma-joined runs-on label list per job queued in . +# Queued runs aren't enough: a run already in progress can have jobs still +# waiting for a runner. Fails when GitHub can't be asked. +queued_jobs() { + local repo="$1" status ids id + for status in queued in_progress; do + ids="$(gh api "repos/$repo/actions/runs?status=$status&per_page=100" \ + --jq '.workflow_runs[].id' 2>/dev/null)" || return 1 + for id in $ids; do + gh api "repos/$repo/actions/runs/$id/jobs?filter=latest&per_page=100" --paginate \ + --jq '.jobs[] | select(.status == "queued") | .labels | map(ascii_downcase) | join(",")' \ + 2>/dev/null || return 1 + done + done +} + +# 's " " line from the idle state, or nothing. +state_line() { + [[ -f "$AUTOSCALE_STATE" ]] || return 0 + awk -v n="$1" '$1 == n' "$AUTOSCALE_STATE" +} + +autoscale_tick() { + local dry="$1" now host decl="" repos repo erepo name min max idle extra + local jobs runners job best bestn cand crepo clabels demand + local line count label mem pids cids cid current busy queued need target reason since state="" + now="$(date +%s)" + host="$(host_label)" + + # " " for every pools.conf pool, autoscaled or not: a + # queued job one of them could take is that pool's to serve. + while IFS= read -r name; do + read -r repo _ extra _ _ <<< "$(conf_line "$name")" + [[ "$extra" == - ]] && extra="" + decl+="$name $repo $(printf 'self-hosted,linux,x64,docker,%s%s' "$host" "${extra:+,$extra}" | tr '[:upper:]' '[:lower:]')"$'\n' + done < <(conf_names) + + # " " per valid autoscale.conf line. + local entries="" + while read -r name min max idle _ || [[ -n ${name:-} ]]; do + [[ -z "$name" || "$name" == \#* ]] && continue + idle="${idle:-10}" + repo="$(awk -v n="$name" '$1 == n { print $2 }' <<< "$decl")" + if [[ -z "$repo" ]]; then + echo "$name: not in $POOLS_CONF -- skipped" >&2; continue + fi + if ! [[ "$min" =~ ^[0-9]+$ && "$max" =~ ^[0-9]+$ && "$idle" =~ ^[0-9]+$ ]] || (( min > max )); then + echo "$name: needs [idle_minutes] as integers, min <= max -- skipped" >&2; continue + fi + entries+="$name $min $max $idle $repo"$'\n' + done < <(tr -d '\r' < "$AUTOSCALE_CONF") + + repos="$(awk 'NF { print $5 }' <<< "$entries" | sort -u)" + while IFS= read -r repo; do + [[ -z "$repo" ]] && continue + if ! jobs="$(queued_jobs "$repo" < /dev/null)" || ! runners="$(repo_runners "$repo" < /dev/null)"; then + while read -r name _ _ _ erepo; do + [[ "$erepo" == "$repo" ]] || continue + echo "$name: could not ask GitHub about $repo -- left as it is" >&2 + # Keep its idle clock: an outage says nothing about whether it's idle. + state+="$(state_line "$name")"$'\n' + done <<< "$entries" + continue + fi + + # Each queued job goes to the declared pool for this repo with the + # fewest labels that still covers it. + demand="" + while IFS= read -r job; do + [[ -n "$job" ]] || continue + best="" bestn=999999 + while read -r cand crepo clabels; do + [[ "$crepo" == "$repo" ]] || continue + labels_cover "$clabels" "$job" || continue + count="$(tr -cd , <<< "$clabels" | wc -c | tr -d ' ')" + if (( count < bestn )); then best="$cand" bestn="$count"; fi + done <<< "$decl" + if [[ -n "$best" ]]; then demand+="$best"$'\n'; fi + done <<< "$jobs" + + while read -r name min max idle erepo; do + [[ "$erepo" == "$repo" ]] || continue + read -r _ _ label mem pids <<< "$(conf_line "$name")" + cids="$(pool_cids "$(project "$name")" < /dev/null)" + current=0 busy=0 + for cid in $cids; do + current=$((current + 1)) + line="$(runner_for "$cid" <<< "$runners")" || continue + if [[ "$(cut -d' ' -f3 <<< "$line")" == true ]]; then busy=$((busy + 1)); fi + done + queued="$(grep -cxF "$name" <<< "$demand" || true)" + need=$((busy + queued)) + target="$current" reason="steady" + if (( current < min )); then + target="$min" reason="below min $min" + fi + if (( need > max )); then need="$max"; fi + if (( need > target )); then + target="$need" reason="$queued queued, $busy busy" + elif (( queued > 0 && need == max && target == current )); then + reason="$queued queued, but at max $max" + fi + if (( target == current && queued == 0 && busy == 0 && current > min )); then + since="$(state_line "$name" | cut -d' ' -f2)" + since="${since:-$now}" + if (( now - since >= idle * 60 )); then + target="$min" reason="idle ${idle}m" + else + state+="$name $since"$'\n' + reason="idle, down to $min in $(( idle * 60 - (now - since) ))s" + fi + fi + echo "$name: $current containers, $busy busy, $queued queued -> $target ($reason)" + [[ -z "$dry" && "$target" != "$current" ]] || continue + # stdin redirected throughout: these run inside a loop reading a here-string. + if (( target < current )) && ! ( require_idle "$name" ) < /dev/null; then + echo "$name: not shrinking after all" >&2 + state+="$name $since"$'\n' + continue + fi + cmd_up "$name" "$repo" "$target" "$label" "$mem" "$pids" < /dev/null \ + || echo "$name: resize to $target failed" >&2 + done <<< "$entries" + done <<< "$repos" + + if [[ -z "$dry" ]]; then + grep -v '^$' <<< "$state" > "$AUTOSCALE_STATE" || true + fi +} + +cmd_autoscale() { + local usage="usage: ./pools.sh autoscale [--once] [--dry-run] [--interval ]" + local once="" dry="" interval=60 + while (( $# )); do + case "$1" in + --once) once=1 ;; + --dry-run) dry=1 ;; + --interval) + [[ "${2:-}" =~ ^[1-9][0-9]*$ ]] || die "$usage -- must be a positive integer" + interval="$2"; shift ;; + *) die "$usage" ;; + esac + shift + done + [[ -f "$AUTOSCALE_CONF" ]] || die "no $AUTOSCALE_CONF here -- copy autoscale.conf.example and list the pools to autoscale" + if [[ -n "$once" ]]; then + autoscale_tick "$dry" + return + fi + while :; do + echo "== $(date -u +%Y-%m-%dT%H:%M:%SZ)" + # A subshell, so a pass that dies (docker down, say) doesn't end the loop. + ( autoscale_tick "$dry" ) || echo "pools.sh: autoscale pass failed; next in ${interval}s" >&2 + sleep "$interval" + done +} + case "${1:-}" in up) shift; cmd_up "$@" ;; down) shift; cmd_down "$@" ;; @@ -620,6 +813,7 @@ case "${1:-}" in scale) shift; cmd_scale "$@" ;; sync) shift; cmd_sync "$@" ;; declare) shift; cmd_declare "$@" ;; + autoscale) shift; cmd_autoscale "$@" ;; list) case "${2:-}" in "") cmd_list ;; diff --git a/tests/pools_test.sh b/tests/pools_test.sh index bf94abe..6151759 100755 --- a/tests/pools_test.sh +++ b/tests/pools_test.sh @@ -60,10 +60,17 @@ esac EOF # One runner for FAKE_CID's container, plus a busy runner on another host -# that must never be counted against this host's pools. +# that must never be counted against this host's pools. For autoscale: one +# queued run whenever FAKE_JOBS is set, whose queued jobs are FAKE_JOBS's +# ;-separated label lists, in every repo. cat > "$tmp/bin/gh" <<'EOF' #!/usr/bin/env bash [[ -n "${FAKE_GH_FAIL:-}" ]] && exit 1 +case "$*" in + *"runs?status=queued"*) if [[ -n "${FAKE_JOBS:-}" ]]; then echo 100; fi; exit 0 ;; + *"runs?status=in_progress"*) exit 0 ;; + *"/jobs"*) tr ';' '\n' <<< "${FAKE_JOBS:-}"; exit 0 ;; +esac echo "somehost-${FAKE_CID:-abc123def456}-42 online ${FAKE_BUSY:-false}" echo "otherhost-999999999999-1 online true" EOF @@ -242,6 +249,99 @@ check "an adopted pool is declared afterwards" 0 \ -- env FAKE_CID=fff000fff000 bash "$s/pools.sh" list --json stray check "sync rejects an unknown option" 1 "usage: ./pools.sh sync" -- bash "$s/pools.sh" sync --bogus +# check_absent -- +check_absent() { + local desc="$1" want_status="$2" unwanted="$3" + shift 4 + local out status=0 + out="$("$@" 2>&1)" || status=$? + if [[ "$status" == "$want_status" && "$out" != *"$unwanted"* ]]; then + echo "ok $desc" + else + echo "FAIL $desc: exit $status (want $want_status), output: $out" + fails=$((fails + 1)) + fi +} + +# autoscale: worm (2 containers, extra label) and plain (none) both serve +# o/r, so a generic job belongs to plain; idle serves o/idle with nothing +# running. The stub containers carry host label "box". +a="$tmp/autoscale" +mkdir -p "$a" +cp "$here/../pools.sh" "$a/" +printf 'worm o/r 2 worm-label 2g 1024\nplain o/r 0\nidle o/idle 0\n' > "$a/pools.conf" +cp "$a/pools.conf" "$a/pools.conf.orig" +as="$a/pools.sh" +export RUNNER_HOST_LABEL=box +worm_job="self-hosted,worm-label" + +check "autoscale refuses without autoscale.conf" 1 "no autoscale.conf" -- bash "$as" autoscale --once +printf '# name min max idle\nworm 1 3 0\nidle 0 2\n' > "$a/autoscale.conf" +check "autoscale rejects an unknown option" 1 "usage: ./pools.sh autoscale" -- bash "$as" autoscale --bogus +check "autoscale rejects a bad interval" 1 "must be a positive integer" -- bash "$as" autoscale --interval 0 + +check "autoscale grows a pool to its busy + queued jobs, capped at max" 0 \ + "worm: 2 containers, 1 busy, 3 queued -> 3 (3 queued, 1 busy)" \ + -- env FAKE_BUSY=true FAKE_JOBS="$worm_job;$worm_job;$worm_job" bash "$as" autoscale --once +check "... through compose, with the pool's pools.conf label and limits" 0 \ + "repo=o/r project=gh-runner-worm label=worm-label mem=2g pids=1024 :: up -d --build --scale runner=3" \ + -- env FAKE_BUSY=true FAKE_JOBS="$worm_job;$worm_job;$worm_job" bash "$as" autoscale --once +check "autoscale doesn't grow for a job its idle containers can take" 0 \ + "worm: 2 containers, 1 busy, 1 queued -> 2 (steady)" \ + -- env FAKE_BUSY=true FAKE_JOBS="$worm_job" bash "$as" autoscale --once +check "a job the plain pool can take doesn't count against the labelled one" 0 \ + "worm: 2 containers, 1 busy, 0 queued -> 2 (steady)" \ + -- env FAKE_BUSY=true FAKE_JOBS="self-hosted,linux" bash "$as" autoscale --once +check "a job for another host's runners counts against nothing here" 0 \ + "worm: 2 containers, 1 busy, 0 queued -> 2 (steady)" \ + -- env FAKE_BUSY=true FAKE_JOBS="self-hosted,otherbox,worm-label" bash "$as" autoscale --once +check "a GitHub-hosted job counts against nothing" 0 \ + "worm: 2 containers, 1 busy, 0 queued -> 2 (steady)" \ + -- env FAKE_BUSY=true FAKE_JOBS="ubuntu-latest" bash "$as" autoscale --once +check "autoscale brings a pool at 0 up for a queued job" 0 \ + "repo=o/idle project=gh-runner-idle label= mem= pids= :: up -d --build --scale runner=1" \ + -- env FAKE_BUSY=true FAKE_JOBS="self-hosted,linux" bash "$as" autoscale --once +check "autoscale shrinks an idle pool to min" 0 \ + "repo=o/r project=gh-runner-worm label=worm-label mem=2g pids=1024 :: up -d --build --scale runner=1" \ + -- bash "$as" autoscale --once +check_absent "... but not when the busy check can't confirm it's idle" 0 ":: up -d" \ + -- env FAKE_CID=000000000000 bash "$as" autoscale --once +check "... and says so" 0 "worm: not shrinking after all" \ + -- env FAKE_CID=000000000000 bash "$as" autoscale --once +check_absent "--dry-run changes nothing" 0 ":: up -d" -- bash "$as" autoscale --once --dry-run +check "... but says what it would do" 0 "worm: 2 containers, 0 busy, 0 queued -> 1 (idle 0m)" \ + -- bash "$as" autoscale --once --dry-run +check_absent "autoscale leaves pools alone when GitHub can't be asked" 0 ":: up -d" \ + -- env FAKE_GH_FAIL=1 bash "$as" autoscale --once +check "... and says so" 0 "worm: could not ask GitHub about o/r -- left as it is" \ + -- env FAKE_GH_FAIL=1 bash "$as" autoscale --once + +printf 'worm 3 5\n' > "$a/autoscale.conf" +check "autoscale raises a pool below min" 0 "worm: 2 containers, 0 busy, 0 queued -> 3 (below min 3)" \ + -- bash "$as" autoscale --once + +printf 'worm 1 2 0\n' > "$a/autoscale.conf" +check "autoscale says when queued jobs meet max" 0 "worm: 2 containers, 1 busy, 3 queued -> 2 (3 queued, but at max 2)" \ + -- env FAKE_BUSY=true FAKE_JOBS="$worm_job;$worm_job;$worm_job" bash "$as" autoscale --once + +printf 'worm 1 3 5\nghost 0 2\nplain 3 1\n' > "$a/autoscale.conf" +rm -f "$a/.autoscale-state" +check "autoscale skips a pool pools.conf doesn't declare" 0 "ghost: not in pools.conf -- skipped" \ + -- bash "$as" autoscale --once +check "autoscale skips a line with min > max" 0 "plain: needs [idle_minutes]" -- bash "$as" autoscale --once +rm -f "$a/.autoscale-state" +check_absent "an idle pool waits out its idle minutes" 0 ":: up -d" -- bash "$as" autoscale --once +check "... counting down" 0 "idle, down to 1 in" -- bash "$as" autoscale --once +check "... remembered between passes" 0 "" -- grep -qE '^worm [0-9]+$' "$a/.autoscale-state" +printf 'worm %s\n' "$(( $(date +%s) - 301 ))" > "$a/.autoscale-state" +check "... then shrinks" 0 ":: up -d --build --scale runner=1" -- bash "$as" autoscale --once +check "... and forgets it" 0 "" -- bash -c "! grep -q worm '$a/.autoscale-state'" +printf 'worm %s\n' "$(( $(date +%s) - 120 ))" > "$a/.autoscale-state" +check "a busy runner resets the idle clock" 0 "worm: 2 containers, 1 busy" -- env FAKE_BUSY=true bash "$as" autoscale --once +check "... so it's forgotten" 0 "" -- bash -c "! grep -q worm '$a/.autoscale-state'" +check "autoscale never rewrites pools.conf" 0 "" -- cmp "$a/pools.conf" "$a/pools.conf.orig" +unset RUNNER_HOST_LABEL + if (( fails )); then echo "$fails check(s) failed" exit 1 From bdcc4411b7fbe926af9d342c34574f53848a0558 Mon Sep 17 00:00:00 2001 From: Steve Leonard Date: Thu, 17 Sep 2026 08:26:36 +0100 Subject: [PATCH 2/2] Address autoscale review: paginate runs, strip .env quotes A repo with over 100 queued or in-progress runs under-counted demand, and a quoted RUNNER_HOST_LABEL in .env never matched the host's own jobs. The README's API-cost claim was optimistic for busy repos. Refs #110 Co-Authored-By: Claude Opus 5 --- README.md | 8 +++++--- pools.sh | 6 ++++-- tests/pools_test.sh | 11 +++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dcb031a..9ea8083 100644 --- a/README.md +++ b/README.md @@ -338,9 +338,11 @@ Every pass, for each listed pool: It resizes with the same `docker compose up --scale` as `scale`, and never writes `pools.conf`: that file's count is still what `start` and `restart` -use, and the next pass corrects it. Each pass reads the runs, their jobs and -the runners of every autoscaled repo -- a few API calls per repo per -minute, well inside the 5,000 an hour a token gets. +use, and the next pass corrects it. Each pass costs, per autoscaled repo, one call +for its runners, one per page of queued and in-progress runs, and one per +such run for its jobs -- so a handful a minute for a quiet repo, but a repo +with 20 runs in flight costs over 20 a minute. A token gets 5,000 an hour; +raise `--interval` if several busy repos are autoscaled. Two hosts serving the same repo each see the same queued job, so each may add a runner for it; the spare one sits idle and goes again after diff --git a/pools.sh b/pools.sh index a59d593..21b4a92 100644 --- a/pools.sh +++ b/pools.sh @@ -632,6 +632,8 @@ host_label() { local label="${RUNNER_HOST_LABEL:-}" if [[ -z "$label" && -f .env ]]; then label="$(tr -d '\r' < .env | sed -n 's/^RUNNER_HOST_LABEL=//p' | tail -1)" + # Compose strips one pair of surrounding quotes; so must this, or the label won't match. + if [[ "$label" =~ ^\"(.*)\"$ || "$label" =~ ^\'(.*)\'$ ]]; then label="${BASH_REMATCH[1]}"; fi fi printf '%s\n' "${label:-unlabelled-host}" } @@ -651,7 +653,7 @@ labels_cover() { queued_jobs() { local repo="$1" status ids id for status in queued in_progress; do - ids="$(gh api "repos/$repo/actions/runs?status=$status&per_page=100" \ + ids="$(gh api "repos/$repo/actions/runs?status=$status&per_page=100" --paginate \ --jq '.workflow_runs[].id' 2>/dev/null)" || return 1 for id in $ids; do gh api "repos/$repo/actions/runs/$id/jobs?filter=latest&per_page=100" --paginate \ @@ -737,7 +739,7 @@ autoscale_tick() { done queued="$(grep -cxF "$name" <<< "$demand" || true)" need=$((busy + queued)) - target="$current" reason="steady" + target="$current" reason="steady" since="$now" if (( current < min )); then target="$min" reason="below min $min" fi diff --git a/tests/pools_test.sh b/tests/pools_test.sh index 6151759..e818760 100755 --- a/tests/pools_test.sh +++ b/tests/pools_test.sh @@ -340,7 +340,18 @@ printf 'worm %s\n' "$(( $(date +%s) - 120 ))" > "$a/.autoscale-state" check "a busy runner resets the idle clock" 0 "worm: 2 containers, 1 busy" -- env FAKE_BUSY=true bash "$as" autoscale --once check "... so it's forgotten" 0 "" -- bash -c "! grep -q worm '$a/.autoscale-state'" check "autoscale never rewrites pools.conf" 0 "" -- cmp "$a/pools.conf" "$a/pools.conf.orig" + unset RUNNER_HOST_LABEL +host_job="self-hosted,box,worm-label" +printf 'RUNNER_HOST_LABEL=box\n' > "$a/.env" +check "autoscale reads the host label from .env" 0 "worm: 2 containers, 1 busy, 1 queued" \ + -- env FAKE_BUSY=true FAKE_JOBS="$host_job" bash "$as" autoscale --once --dry-run +printf 'RUNNER_HOST_LABEL="box"\n' > "$a/.env" +check "... with the quotes compose would strip" 0 "worm: 2 containers, 1 busy, 1 queued" \ + -- env FAKE_BUSY=true FAKE_JOBS="$host_job" bash "$as" autoscale --once --dry-run +rm -f "$a/.env" +check "... and none without one" 0 "worm: 2 containers, 1 busy, 0 queued" \ + -- env FAKE_BUSY=true FAKE_JOBS="$host_job" bash "$as" autoscale --once --dry-run if (( fails )); then echo "$fails check(s) failed"