Skip to content

Commit 9d2f0f8

Browse files
sikienzlsikienzl
committed
feat(monitoring): add benchmark AI grafana tile
1 parent 4a4f179 commit 9d2f0f8

2 files changed

Lines changed: 48 additions & 5 deletions

File tree

‎deploy/grafana-dashboard.json‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
}
5252
},
5353
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 44 },
54-
"id": 29,
54+
"id": 31,
5555
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "value_and_name" },
5656
"title": "Portfolio Snapshot Age",
5757
"type": "stat",
@@ -444,7 +444,7 @@
444444
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "value" },
445445
"title": "AI Monthly Calls",
446446
"type": "stat",
447-
"targets": [{ "expr": "trading_ai_copilot_calls_used_monthly", "legendFormat": "Monthly", "refId": "A" }]
447+
"targets": [{ "expr": "trading_ai_copilot_monthly_calls", "legendFormat": "Monthly", "refId": "A" }]
448448
},
449449
{
450450
"datasource": "Prometheus",
@@ -464,7 +464,27 @@
464464
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "value" },
465465
"title": "Mammouth Spend",
466466
"type": "stat",
467-
"targets": [{ "expr": "trading_ai_copilot_budget_used_usd", "legendFormat": "Spend", "refId": "A" }]
467+
"targets": [{ "expr": "trading_ai_copilot_monthly_spend_usd", "legendFormat": "Spend", "refId": "A" }]
468+
},
469+
{
470+
"datasource": "Prometheus",
471+
"fieldConfig": {
472+
"defaults": {
473+
"color": { "mode": "thresholds" },
474+
"thresholds": { "mode": "absolute", "steps": [
475+
{ "color": "green", "value": null },
476+
{ "color": "yellow", "value": 0.25 },
477+
{ "color": "red", "value": 0.75 }
478+
] },
479+
"unit": "currencyUSD"
480+
}
481+
},
482+
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 44 },
483+
"id": 30,
484+
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "value" },
485+
"title": "Benchmark Spend",
486+
"type": "stat",
487+
"targets": [{ "expr": "trading_ai_copilot_benchmark_monthly_spend_usd", "legendFormat": "Benchmark", "refId": "A" }]
468488
},
469489
{
470490
"datasource": "Prometheus",
@@ -519,7 +539,7 @@
519539
"unit": "currencyEUR"
520540
}
521541
},
522-
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 44 },
542+
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 44 },
523543
"id": 22,
524544
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "value_and_name" },
525545
"title": "Portfolio Value",
@@ -538,7 +558,7 @@
538558
"unit": "currencyEUR"
539559
}
540560
},
541-
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 44 },
561+
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 44 },
542562
"id": 23,
543563
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "value_and_name" },
544564
"title": "Portfolio Start",

‎scripts/run_weekly_scorecard.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,7 @@ if [[ "${STATUS_PROM_ENABLED,,}" == "true" ]]; then
394394
STATUS_PROM_DIR="$(dirname "$STATUS_PROM_FILE")"
395395
mkdir -p "$STATUS_PROM_DIR"
396396
AI_STATE_FILE="${AI_COPILOT_STATE_FILE:-$ROOT_DIR/ai_copilot_state.json}"
397+
AI_BENCHMARK_STATE_FILE="${AI_COPILOT_BENCHMARK_STATE_FILE:-$ROOT_DIR/ai_copilot_benchmark_state.json}"
397398
PORTFOLIO_STATE_FILE="${PORTFOLIO_STATE_FILE:-$ROOT_DIR/.portfolio_state.json}"
398399
BOT_LOG_FILE="${BOT_LOG_FILE:-$ROOT_DIR/logs/bot.log}"
399400

@@ -445,6 +446,7 @@ if [[ "${STATUS_PROM_ENABLED,,}" == "true" ]]; then
445446
STATUS_TS_UNIX="$TS_UNIX" \
446447
SCORECARD_STARTING_CAPITAL="$STARTING_CAPITAL" \
447448
AI_STATE_FILE="$AI_STATE_FILE" \
449+
AI_BENCHMARK_STATE_FILE="$AI_BENCHMARK_STATE_FILE" \
448450
PORTFOLIO_STATE_FILE="$PORTFOLIO_STATE_FILE" \
449451
BOT_LOG_FILE="$BOT_LOG_FILE" \
450452
"$PYTHON_CMD" - <<'PY'
@@ -542,6 +544,7 @@ def _read_portfolio_values_from_log(log_path: str):
542544
543545
544546
ai_state = _read_json(os.environ["AI_STATE_FILE"])
547+
ai_benchmark_state = _read_json(os.environ["AI_BENCHMARK_STATE_FILE"])
545548
portfolio_state = _read_json(os.environ["PORTFOLIO_STATE_FILE"])
546549
session_start_value, latest_portfolio_value = _read_portfolio_values_from_log(os.environ["BOT_LOG_FILE"])
547550
portfolio_value = latest_portfolio_value
@@ -561,6 +564,14 @@ budget_used_pct = 0.0
561564
if monthly_budget > 0:
562565
budget_used_pct = (monthly_spend / monthly_budget) * 100.0
563566
567+
benchmark_monthly_calls = int(_as_float(ai_benchmark_state.get("monthly_calls"), 0.0))
568+
benchmark_daily_calls = int(_as_float(ai_benchmark_state.get("daily_calls"), 0.0))
569+
benchmark_monthly_spend = _as_float(ai_benchmark_state.get("monthly_spend_usd"), 0.0)
570+
benchmark_monthly_budget = _as_float(ai_benchmark_state.get("budget_cap_usd"), 0.0)
571+
benchmark_budget_used_pct = 0.0
572+
if benchmark_monthly_budget > 0:
573+
benchmark_budget_used_pct = (benchmark_monthly_spend / benchmark_monthly_budget) * 100.0
574+
564575
lines = [
565576
"# HELP trading_ai_copilot_daily_calls AI copilot calls used in the current day",
566577
"# TYPE trading_ai_copilot_daily_calls gauge",
@@ -574,6 +585,18 @@ lines = [
574585
"# HELP trading_ai_copilot_budget_used_pct Estimated percentage of AI copilot monthly budget already used",
575586
"# TYPE trading_ai_copilot_budget_used_pct gauge",
576587
f"trading_ai_copilot_budget_used_pct {budget_used_pct:.4f}",
588+
"# HELP trading_ai_copilot_benchmark_daily_calls Benchmark AI copilot calls used in the current day",
589+
"# TYPE trading_ai_copilot_benchmark_daily_calls gauge",
590+
f"trading_ai_copilot_benchmark_daily_calls {benchmark_daily_calls}",
591+
"# HELP trading_ai_copilot_benchmark_monthly_calls Benchmark AI copilot calls used in the current month",
592+
"# TYPE trading_ai_copilot_benchmark_monthly_calls gauge",
593+
f"trading_ai_copilot_benchmark_monthly_calls {benchmark_monthly_calls}",
594+
"# HELP trading_ai_copilot_benchmark_monthly_spend_usd Estimated benchmark AI copilot spend in USD for the current month",
595+
"# TYPE trading_ai_copilot_benchmark_monthly_spend_usd gauge",
596+
f"trading_ai_copilot_benchmark_monthly_spend_usd {benchmark_monthly_spend:.6f}",
597+
"# HELP trading_ai_copilot_benchmark_budget_used_pct Estimated percentage of benchmark AI copilot monthly budget already used",
598+
"# TYPE trading_ai_copilot_benchmark_budget_used_pct gauge",
599+
f"trading_ai_copilot_benchmark_budget_used_pct {benchmark_budget_used_pct:.4f}",
577600
"# HELP trading_runtime_portfolio_value Latest portfolio value seen by the trading bot",
578601
"# TYPE trading_runtime_portfolio_value gauge",
579602
f"trading_runtime_portfolio_value {portfolio_value:.6f}",

0 commit comments

Comments
 (0)