Improve dashboard live updates
This commit is contained in:
1 parent
9f868fca89
commit
43e417c444
2 files changed
+401
-92
No files matched your search
@@ -1011,9 +1011,10 @@ cmd_exec() {
|
||||
status_path="$REMOTE_ROOT/.autodl-remote/jobs/$name.status"
|
||||
info "log=$log"
|
||||
if [ "$detach" = "true" ]; then
|
||||
local runner
|
||||
local runner runner_local
|
||||
runner="$REMOTE_ROOT/.autodl-remote/tmp/$name.runner.sh"
|
||||
upload_stdin_to_remote "$runner" <<EOF
|
||||
runner_local="$(mktemp "${TMPDIR:-/tmp}/autodl-remote-runner.XXXXXX")" || die "mktemp failed"
|
||||
cat > "$runner_local" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set +e
|
||||
started_at=\$(date +%Y-%m-%dT%H:%M:%S%z)
|
||||
@@ -1025,6 +1026,8 @@ finished_at=\$(date +%Y-%m-%dT%H:%M:%S%z)
|
||||
printf 'status=exited\npid=%s\nstarted_at=%s\nfinished_at=%s\nexit_code=%s\nlog=%s\ncwd=%s\n' "\$pid" "\$started_at" "\$finished_at" "\$rc" $(shell_quote "$log") $(shell_quote "$cwd") > $(shell_quote "$status_path")
|
||||
exit "\$rc"
|
||||
EOF
|
||||
upload_local_file "$runner_local" "$runner" || { rm -f "$runner_local"; return 1; }
|
||||
rm -f "$runner_local"
|
||||
remote_cmd="mkdir -p $(shell_quote "$(dirname "$log")") $(shell_quote "$(dirname "$status_path")") && chmod +x $(shell_quote "$runner") && if command -v setsid >/dev/null 2>&1; then setsid bash $(shell_quote "$runner") < /dev/null > /dev/null 2>&1 & else nohup bash $(shell_quote "$runner") < /dev/null > /dev/null 2>&1 & fi; printf 'pid=%s\nlog=%s\nstatus=%s\n' \"\$!\" $(shell_quote "$log") $(shell_quote "$status_path")"
|
||||
result="$(ssh_capture "bash -lc $(shell_quote "$remote_cmd")" 2>&1)"
|
||||
detach_rc=$?
|
||||
@@ -1531,27 +1534,6 @@ cmd_fleet() {
|
||||
esac
|
||||
}
|
||||
|
||||
dashboard_device_row() {
|
||||
local device="$1" account="$2" remote="$3" tags="$4" probe online gpu disk current esc
|
||||
load_account "$account"
|
||||
REMOTE_ROOT="${remote%/}"
|
||||
validate_remote_root "$REMOTE_ROOT"
|
||||
probe="$(remote_probe)"
|
||||
online="$(kv_get "$probe" online)"
|
||||
gpu="$(kv_get "$probe" gpu)"
|
||||
disk="$(kv_get "$probe" disk)"
|
||||
current="$(latest_run_for_device "$account" "$REMOTE_ROOT")"
|
||||
printf '<tr><td>%s</td><td class="%s">%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n' \
|
||||
"$(html_escape "$device")" \
|
||||
"$([ "${online:-no}" = "yes" ] && printf online || printf offline)" \
|
||||
"$(html_escape "${online:-no}")" \
|
||||
"$(html_escape "$account")" \
|
||||
"$(html_escape "${gpu:--}")" \
|
||||
"$(html_escape "${disk:--}")" \
|
||||
"$(html_escape "$current")" \
|
||||
"$(html_escape "${tags:--}")"
|
||||
}
|
||||
|
||||
remote_log_tail() {
|
||||
local account="$1" remote="$2" log="$3" lines="$4" output rc
|
||||
[ -n "$log" ] || { printf '[log missing]\n'; return 0; }
|
||||
@@ -1567,58 +1549,339 @@ remote_log_tail() {
|
||||
fi
|
||||
}
|
||||
|
||||
dashboard_run_rows() {
|
||||
local scope_fleet="$1" lines="$2" dir file name account remote model stage tags started status log purpose include log_tail
|
||||
dashboard_pct() {
|
||||
local pct
|
||||
pct="$(printf '%s\n' "$1" | awk 'match($0, /[0-9]+%/) { print substr($0, RSTART, RLENGTH - 1); exit }')"
|
||||
case "$pct" in
|
||||
""|*[!0-9]*) pct=0 ;;
|
||||
esac
|
||||
[ "$pct" -gt 100 ] 2>/dev/null && pct=100
|
||||
printf '%s\n' "$pct"
|
||||
}
|
||||
|
||||
dashboard_gpu_model() {
|
||||
local value="$1" model
|
||||
model="$(printf '%s\n' "$value" | sed -E 's/[[:space:]][0-9]+\/[0-9]+ MiB [0-9]+%$//')"
|
||||
[ -n "$model" ] || model="-"
|
||||
printf '%s\n' "$model"
|
||||
}
|
||||
|
||||
dashboard_gpu_mem() {
|
||||
local value="$1" mem
|
||||
mem="$(printf '%s\n' "$value" | sed -nE 's/.* ([0-9]+\/[0-9]+ MiB) [0-9]+%$/\1/p')"
|
||||
[ -n "$mem" ] || mem="-"
|
||||
printf '%s\n' "$mem"
|
||||
}
|
||||
|
||||
dashboard_status_class() {
|
||||
local online="$1" status="$2"
|
||||
if [ "${online:-no}" != "yes" ]; then
|
||||
printf 'offline\n'
|
||||
return 0
|
||||
fi
|
||||
case "$status" in
|
||||
running) printf 'training\n' ;;
|
||||
unreachable|failed|error) printf 'error\n' ;;
|
||||
*) printf 'idle\n' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
dashboard_status_label() {
|
||||
case "$1" in
|
||||
training) printf '训练中\n' ;;
|
||||
idle) printf '空闲\n' ;;
|
||||
error) printf '异常\n' ;;
|
||||
offline) printf '离线\n' ;;
|
||||
*) printf '%s\n' "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
dashboard_job_in_scope() {
|
||||
local file="$1" scope_fleet="$2" account remote fleet_file fleet_account fleet_remote
|
||||
account="$(job_get "$file" ACCOUNT)"
|
||||
remote="$(job_get "$file" REMOTE_ROOT)"
|
||||
[ -n "$account" ] || account="${PROJECT_BOUND_ACCOUNT:-$PROJECT_ACCOUNT}"
|
||||
[ -n "$remote" ] || remote="${PROJECT_BOUND_REMOTE_ROOT:-$REMOTE_ROOT}"
|
||||
if [ -z "$scope_fleet" ]; then
|
||||
[ "$account" = "${PROJECT_BOUND_ACCOUNT:-$PROJECT_ACCOUNT}" ] && [ "$remote" = "${PROJECT_BOUND_REMOTE_ROOT:-$REMOTE_ROOT}" ]
|
||||
return $?
|
||||
fi
|
||||
for fleet_file in "$(fleet_dir "$scope_fleet")"/*.conf; do
|
||||
[ -f "$fleet_file" ] || continue
|
||||
fleet_account="$(device_get "$fleet_file" ACCOUNT)"
|
||||
fleet_remote="$(device_get "$fleet_file" REMOTE_ROOT)"
|
||||
if [ "$fleet_account" = "$account" ] && [ "$fleet_remote" = "$remote" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
latest_job_file_for_device() {
|
||||
local account="$1" remote="$2" dir file latest_file="" latest_started="" started job_account job_remote
|
||||
dir="$(jobs_dir)"
|
||||
if ls "$dir"/*.conf >/dev/null 2>&1; then
|
||||
for file in "$dir"/*.conf; do
|
||||
job_account="$(job_get "$file" ACCOUNT)"
|
||||
job_remote="$(job_get "$file" REMOTE_ROOT)"
|
||||
[ -n "$job_account" ] || job_account="${PROJECT_BOUND_ACCOUNT:-$PROJECT_ACCOUNT}"
|
||||
[ -n "$job_remote" ] || job_remote="${PROJECT_BOUND_REMOTE_ROOT:-$REMOTE_ROOT}"
|
||||
[ "$job_account" = "$account" ] || continue
|
||||
[ "$job_remote" = "$remote" ] || continue
|
||||
started="$(job_get "$file" STARTED_AT)"
|
||||
if [ -z "$latest_started" ] || [ "$started" \> "$latest_started" ]; then
|
||||
latest_started="$started"
|
||||
latest_file="$file"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
printf '%s\n' "$latest_file"
|
||||
}
|
||||
|
||||
dashboard_device_card() {
|
||||
local device="$1" account="$2" remote="$3" tags="$4" lines="$5"
|
||||
local probe online host gpu disk processes current_file current status status_class status_label
|
||||
local gpu_util gpu_model gpu_mem disk_pct log log_tail model stage purpose started card_id
|
||||
|
||||
DASHBOARD_DEVICE_TOTAL=$((DASHBOARD_DEVICE_TOTAL + 1))
|
||||
load_account "$account"
|
||||
REMOTE_ROOT="${remote%/}"
|
||||
validate_remote_root "$REMOTE_ROOT"
|
||||
probe="$(remote_probe)"
|
||||
online="$(kv_get "$probe" online)"
|
||||
host="$(kv_get "$probe" host)"
|
||||
gpu="$(kv_get "$probe" gpu)"
|
||||
disk="$(kv_get "$probe" disk)"
|
||||
processes="$(kv_get "$probe" processes)"
|
||||
current_file="$(latest_job_file_for_device "$account" "$REMOTE_ROOT")"
|
||||
current="-"
|
||||
status="idle"
|
||||
log=""
|
||||
log_tail="[no recorded run]"
|
||||
model="-"
|
||||
stage="-"
|
||||
purpose=""
|
||||
started="-"
|
||||
if [ -n "$current_file" ]; then
|
||||
current="$(job_get "$current_file" NAME)"
|
||||
status="$(job_remote_status_summary "$current_file")"
|
||||
log="$(job_get "$current_file" LOG_PATH)"
|
||||
model="$(job_get "$current_file" MODEL)"
|
||||
stage="$(job_get "$current_file" STAGE)"
|
||||
purpose="$(job_get "$current_file" PURPOSE)"
|
||||
started="$(job_get "$current_file" STARTED_AT)"
|
||||
[ -n "$model" ] || model="-"
|
||||
[ -n "$stage" ] || stage="-"
|
||||
if [ "$lines" -gt 0 ]; then
|
||||
log_tail="$(remote_log_tail "$account" "$REMOTE_ROOT" "$log" "$lines")"
|
||||
else
|
||||
log_tail="[log hidden: --lines 0]"
|
||||
fi
|
||||
elif [ -n "$processes" ]; then
|
||||
log_tail="$processes"
|
||||
fi
|
||||
|
||||
status_class="$(dashboard_status_class "$online" "$status")"
|
||||
status_label="$(dashboard_status_label "$status_class")"
|
||||
case "$status_class" in
|
||||
training) DASHBOARD_TRAINING_TOTAL=$((DASHBOARD_TRAINING_TOTAL + 1)) ;;
|
||||
idle) DASHBOARD_IDLE_TOTAL=$((DASHBOARD_IDLE_TOTAL + 1)) ;;
|
||||
error) DASHBOARD_ERROR_TOTAL=$((DASHBOARD_ERROR_TOTAL + 1)) ;;
|
||||
offline) DASHBOARD_OFFLINE_TOTAL=$((DASHBOARD_OFFLINE_TOTAL + 1)) ;;
|
||||
esac
|
||||
[ "${online:-no}" = "yes" ] && DASHBOARD_ONLINE_TOTAL=$((DASHBOARD_ONLINE_TOTAL + 1))
|
||||
|
||||
gpu="${gpu:--}"
|
||||
disk="${disk:--}"
|
||||
gpu_util="$(dashboard_pct "$gpu")"
|
||||
disk_pct="$(dashboard_pct "$disk")"
|
||||
gpu_model="$(dashboard_gpu_model "$gpu")"
|
||||
gpu_mem="$(dashboard_gpu_mem "$gpu")"
|
||||
card_id="$(sanitize_name "${device}-${account}")"
|
||||
DASHBOARD_HEAT_CELLS="${DASHBOARD_HEAT_CELLS}<span class=\"heat-cell heat-${status_class}\" title=\"$(html_escape "$device · ${gpu_util}%")\" style=\"--v:${gpu_util}%\"></span>"
|
||||
|
||||
printf '<article class="mc mc-%s" data-status="%s">\n' "$status_class" "$status_class"
|
||||
printf '<div class="mc-head"><div class="mc-head-l"><div class="mc-id"><span class="mc-bracket">[</span><span class="mc-id-text">%s</span><span class="mc-bracket">]</span></div><div class="mc-name">%s</div><div class="mc-sub"><span>%s</span><span class="sep">·</span><span>%s</span></div></div><div class="mc-head-r"><span class="st-badge"><span class="st-dot"></span>%s</span><div class="mc-uptime">%s</div></div></div>\n' \
|
||||
"$(html_escape "$device")" "$(html_escape "$account")" "$(html_escape "${host:--}")" "$(html_escape "${tags:--}")" "$(html_escape "$status_label")" "$(html_escape "${status:-unknown}")"
|
||||
printf '<div class="mc-agg"><div class="mc-agg-l"><span class="agg-k">GPU UTIL</span><span class="agg-v">%s<span class="pct-s">%%</span></span></div><div class="ubar-wrap"><div class="ubar-track"></div><div class="ubar-fill" style="width:%s%%"></div></div><div class="mc-agg-r">%s</div></div>\n' \
|
||||
"$(html_escape "$gpu_util")" "$(html_escape "$gpu_util")" "$(html_escape "$gpu_mem")"
|
||||
printf '<div class="gpu-row"><div class="gpu-row-head"><span class="gpu-tag">GPU</span><span class="gpu-model">%s</span><span class="gpu-util-pct">%s<span class="pct-s">%%</span></span></div><div class="gpu-row-meta"><span class="meta-cell"><span class="meta-k">DISK</span><span class="meta-v">%s</span><span class="meta-mini-bar"><span style="width:%s%%"></span></span></span><span class="meta-cell"><span class="meta-k">REMOTE</span><span class="meta-v">%s</span></span></div></div>\n' \
|
||||
"$(html_escape "$gpu_model")" "$(html_escape "$gpu_util")" "$(html_escape "$disk")" "$(html_escape "$disk_pct")" "$(html_escape "$REMOTE_ROOT")"
|
||||
printf '<div class="mc-task"><div class="task-row"><span class="task-arrow">▸</span><span class="task-name">%s</span></div><div class="task-meta">%s · %s · %s</div>' \
|
||||
"$(html_escape "$current")" "$(html_escape "$model")" "$(html_escape "$stage")" "$(html_escape "$started")"
|
||||
[ -n "$purpose" ] && printf '<div class="task-purpose">%s</div>' "$(html_escape "$purpose")"
|
||||
printf '</div>\n'
|
||||
printf '<div class="mc-term"><div class="term-bar"><span class="term-dot"></span><span class="term-bar-t">%s</span><span class="term-bar-eta">last %s</span></div><pre class="term-body" data-term-id="%s">%s</pre></div>\n' \
|
||||
"$(html_escape "${log:-processes}")" "$(html_escape "$lines")" "$(html_escape "$card_id")" "$(html_escape "$log_tail")"
|
||||
printf '</article>\n'
|
||||
}
|
||||
|
||||
dashboard_device_cards() {
|
||||
local fleet="$1" lines="$2" device_file device account remote tags
|
||||
if [ -n "$fleet" ]; then
|
||||
for device_file in "$(fleet_dir "$fleet")"/*.conf; do
|
||||
[ -f "$device_file" ] || continue
|
||||
device="$(device_get "$device_file" NAME)"
|
||||
account="$(device_get "$device_file" ACCOUNT)"
|
||||
remote="$(device_get "$device_file" REMOTE_ROOT)"
|
||||
tags="$(device_get "$device_file" TAGS)"
|
||||
dashboard_device_card "$device" "$account" "$remote" "$tags" "$lines"
|
||||
done
|
||||
else
|
||||
dashboard_device_card "current" "$PROJECT_ACCOUNT" "$REMOTE_ROOT" "project" "$lines"
|
||||
fi
|
||||
}
|
||||
|
||||
dashboard_run_tiles() {
|
||||
local scope_fleet="$1" dir file name account remote model stage started log purpose
|
||||
dir="$(jobs_dir)"
|
||||
printf '<section class="run-panel"><div class="sec-t"><span class="sec-t-l">─ RUNS</span><span class="sec-t-line"></span></div><div class="run-list">\n'
|
||||
if ls "$dir"/*.conf >/dev/null 2>&1; then
|
||||
for file in "$dir"/*.conf; do
|
||||
dashboard_job_in_scope "$file" "$scope_fleet" || continue
|
||||
DASHBOARD_RUN_TOTAL=$((DASHBOARD_RUN_TOTAL + 1))
|
||||
name="$(job_get "$file" NAME)"
|
||||
account="$(job_get "$file" ACCOUNT)"
|
||||
remote="$(job_get "$file" REMOTE_ROOT)"
|
||||
model="$(job_get "$file" MODEL)"
|
||||
stage="$(job_get "$file" STAGE)"
|
||||
tags="$(job_get "$file" TAGS)"
|
||||
started="$(job_get "$file" STARTED_AT)"
|
||||
log="$(job_get "$file" LOG_PATH)"
|
||||
purpose="$(job_get "$file" PURPOSE)"
|
||||
[ -n "$account" ] || account="${PROJECT_BOUND_ACCOUNT:-$PROJECT_ACCOUNT}"
|
||||
[ -n "$remote" ] || remote="${PROJECT_BOUND_REMOTE_ROOT:-$REMOTE_ROOT}"
|
||||
include=true
|
||||
if [ -z "$scope_fleet" ]; then
|
||||
if [ "$account" != "${PROJECT_BOUND_ACCOUNT:-$PROJECT_ACCOUNT}" ] || [ "$remote" != "${PROJECT_BOUND_REMOTE_ROOT:-$REMOTE_ROOT}" ]; then
|
||||
include=false
|
||||
fi
|
||||
else
|
||||
include=false
|
||||
local fleet_file fleet_account fleet_remote
|
||||
for fleet_file in "$(fleet_dir "$scope_fleet")"/*.conf; do
|
||||
[ -f "$fleet_file" ] || continue
|
||||
fleet_account="$(device_get "$fleet_file" ACCOUNT)"
|
||||
fleet_remote="$(device_get "$fleet_file" REMOTE_ROOT)"
|
||||
if [ "$fleet_account" = "$account" ] && [ "$fleet_remote" = "$remote" ]; then
|
||||
include=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
[ "$include" = "true" ] || continue
|
||||
status="$(job_remote_status_summary "$file")"
|
||||
printf '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td><code>%s</code><div class="muted">%s</div></td></tr>\n' \
|
||||
"$(html_escape "$name")" "$(html_escape "$account")" "$(html_escape "$status")" \
|
||||
"$(html_escape "${model:--}")" "$(html_escape "${stage:--}")" "$(html_escape "$started")" \
|
||||
"$(html_escape "${tags:--}")" "$(html_escape "$log")" "$(html_escape "$purpose")"
|
||||
if [ "$lines" -gt 0 ]; then
|
||||
log_tail="$(remote_log_tail "$account" "$remote" "$log" "$lines")"
|
||||
printf '<tr class="log-row"><td colspan="8"><div class="log-title">%s · last %s lines</div><pre class="log">%s</pre></td></tr>\n' \
|
||||
"$(html_escape "$name")" "$(html_escape "$lines")" "$(html_escape "$log_tail")"
|
||||
fi
|
||||
printf '<div class="run-chip"><div class="run-chip-h"><span>%s</span><span>%s</span></div><div class="run-chip-m">%s · %s · %s</div><div class="run-chip-f">%s</div><div class="run-chip-note">%s</div></div>\n' \
|
||||
"$(html_escape "$name")" "$(html_escape "${stage:--}")" "$(html_escape "$account")" "$(html_escape "${model:--}")" "$(html_escape "$started")" "$(html_escape "$log")" "$(html_escape "$purpose")"
|
||||
done
|
||||
fi
|
||||
if [ "$DASHBOARD_RUN_TOTAL" -eq 0 ]; then
|
||||
printf '<div class="empty">NO RUNS RECORDED</div>\n'
|
||||
fi
|
||||
printf '</div></section>\n'
|
||||
}
|
||||
|
||||
render_dashboard() {
|
||||
local fleet="$1" output="$2" lines="$3" refresh_seconds="${4:-}" generated_at title
|
||||
mkdir -p "$(dirname "$output")"
|
||||
dashboard_state_path() {
|
||||
local output="$1" dir base
|
||||
dir="$(dirname "$output")"
|
||||
base="$(basename "$output")"
|
||||
case "$base" in
|
||||
*.html) base="${base%.html}.state.js" ;;
|
||||
*) base="$base.state.js" ;;
|
||||
esac
|
||||
printf '%s/%s\n' "$dir" "$base"
|
||||
}
|
||||
|
||||
js_string() {
|
||||
printf '"'
|
||||
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
||||
printf '"'
|
||||
}
|
||||
|
||||
base64_file_compact() {
|
||||
base64 < "$1" | tr -d '\n'
|
||||
}
|
||||
|
||||
dashboard_css() {
|
||||
cat <<'EOF'
|
||||
:root{--bg:#fdf6e3;--bg-2:#f3ead0;--card:#fffbef;--card-2:#f7efd5;--border:#e1d6a8;--border-2:#cdc295;--border-soft:#ebe1b8;--fg:#073642;--fg-dim:#586e75;--dim:#93a1a1;--muted:#93a1a1;--muted-strong:#657b83;--ok:#859900;--hot:#dc322f;--warm:#b58900;--cool:#268bd2;--err:#dc322f;--off:#93a1a1;--off-bg:#e1d6a8;--term-bg:#002b36;--term-fg:#93a1a1}*{box-sizing:border-box}html,body{margin:0;min-height:100%;background:var(--bg);color:var(--fg)}body{font-family:"JetBrains Mono","IBM Plex Mono",ui-monospace,SFMono-Regular,Menlo,monospace;-webkit-font-smoothing:antialiased;overflow-x:hidden}.app{min-height:100vh;background:radial-gradient(ellipse 1200px 600px at 50% -200px,rgba(133,153,0,.09),transparent 70%),linear-gradient(var(--bg),var(--bg-2));display:flex;flex-direction:column}.scanlines{position:fixed;inset:0;pointer-events:none;z-index:50;background-image:repeating-linear-gradient(to bottom,rgba(0,0,0,0) 0,rgba(0,0,0,0) 2px,rgba(0,0,0,.025) 3px,rgba(0,0,0,0) 4px);mix-blend-mode:multiply}.vignette{position:fixed;inset:0;pointer-events:none;z-index:49;box-shadow:inset 0 0 180px rgba(0,0,0,.08)}.topbar{position:sticky;top:0;z-index:30;background:rgba(253,246,227,.88);backdrop-filter:blur(10px) saturate(150%);border-bottom:1px solid var(--border);padding:14px 22px 0;display:grid;grid-template-columns:minmax(280px,1fr) minmax(420px,1.4fr) minmax(260px,1fr);grid-template-rows:auto auto;grid-template-areas:"l c r" "f f f";gap:16px 24px;align-items:center}.topbar-l{grid-area:l;display:flex;flex-direction:column;gap:6px}.topbar-c{grid-area:c;display:flex;justify-content:center}.topbar-r{grid-area:r;display:flex;justify-content:flex-end;align-items:center;gap:20px}.filters{grid-area:f;display:flex;gap:8px;padding:10px 0;border-top:1px dashed var(--border-2);align-items:center}.brand{display:flex;align-items:center;gap:10px}.brand-mark{width:22px;height:22px;display:inline-grid;place-items:center;background:var(--ok);color:var(--bg);font-size:14px;border-radius:3px;box-shadow:0 0 12px rgba(133,153,0,.38)}.brand-name{font-size:17px;font-weight:700;letter-spacing:.14em}.brand-sub{font-size:11px;font-weight:600;letter-spacing:.22em;color:var(--fg-dim);padding:2px 6px;border:1px solid var(--border);border-radius:2px}.brand-ver{font-size:10px;color:var(--muted);margin-left:4px}.brand-tagline{font-size:11px;color:var(--fg-dim);letter-spacing:.02em}.agg-block{width:100%;max-width:560px;border:1px solid var(--border);background:var(--card);padding:8px 12px 9px;border-radius:4px}.agg-block-h{display:flex;justify-content:space-between;font-size:10.5px;letter-spacing:.12em;margin-bottom:6px}.heatmap{display:flex;gap:2px;height:16px;margin-bottom:6px}.heat-cell{display:block;flex:1;border-radius:1px;background:var(--off-bg);opacity:.65}.heat-training{background:var(--ok);opacity:.9}.heat-idle{background:var(--cool);opacity:.45}.heat-error{background:var(--err);opacity:.85}.heat-offline{background:var(--off-bg);opacity:.7}.agg-block-f{font-size:10.5px;letter-spacing:.04em;color:var(--dim)}.clock{text-align:right}.clock-t{font-size:22px;font-weight:600;letter-spacing:.06em;font-variant-numeric:tabular-nums}.clock-d{font-size:10.5px;letter-spacing:.08em;margin-top:2px;color:var(--dim)}.refresh{display:flex;flex-direction:column;gap:4px;align-items:flex-end}.refresh>.dim{font-size:10px;letter-spacing:.14em;color:var(--dim)}.seg{display:flex;border:1px solid var(--border);border-radius:3px;overflow:hidden}.seg-btn{appearance:none;background:transparent;border:0;padding:4px 10px;color:var(--fg-dim);font:inherit;font-size:11px;border-right:1px solid var(--border);font-variant-numeric:tabular-nums}.seg-btn:last-child{border-right:0}.seg-btn.on{color:var(--bg);background:var(--ok)}.flt{appearance:none;background:transparent;border:1px solid var(--border);color:var(--fg-dim);font:inherit;font-size:11px;padding:5px 12px;cursor:pointer;border-radius:3px;display:inline-flex;align-items:center;gap:8px;letter-spacing:.02em}.flt-c{color:var(--muted);font-variant-numeric:tabular-nums;font-size:10px}.flt:hover{color:var(--fg);border-color:var(--fg-dim)}.flt.on{color:var(--bg);background:var(--ok);border-color:var(--ok)}.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(380px,1fr));gap:14px;padding:18px 22px 26px;flex:1}.mc{background:var(--card);border:1px solid var(--border);border-radius:4px;padding:14px 14px 0;display:flex;flex-direction:column;gap:12px;position:relative;overflow:hidden}.mc::before{content:"";position:absolute;left:0;top:0;bottom:0;width:2px;background:var(--accent,var(--ok));opacity:.85}.mc-training{--accent:var(--ok)}.mc-idle{--accent:var(--cool)}.mc-error{--accent:var(--err);background:linear-gradient(180deg,rgba(220,50,47,.06),var(--card))}.mc-offline{--accent:var(--off);opacity:.72}.mc-head{display:flex;justify-content:space-between;align-items:flex-start;gap:12px}.mc-head-l{display:flex;flex-direction:column;gap:3px;min-width:0}.mc-head-r{display:flex;flex-direction:column;align-items:flex-end;gap:4px}.mc-id{fLine truncated
|
||||
EOF
|
||||
}
|
||||
|
||||
dashboard_js() {
|
||||
cat <<'EOF'
|
||||
(function(){
|
||||
const pageKey = "autodl-dashboard:" + location.pathname;
|
||||
const stateSrc = window.__AUTODL_STATE_SRC__;
|
||||
const refreshSeconds = Number(window.__AUTODL_REFRESH_SECONDS__ || 0);
|
||||
let currentFilter = sessionStorage.getItem(pageKey + ":filter") || "all";
|
||||
function atBottom(el){ return el.scrollHeight - el.scrollTop - el.clientHeight < 24; }
|
||||
function captureTerms(){
|
||||
const terms = {};
|
||||
document.querySelectorAll(".term-body").forEach((el) => {
|
||||
const id = el.dataset.termId || "";
|
||||
terms[id] = { top: el.scrollTop, follow: atBottom(el) };
|
||||
});
|
||||
return terms;
|
||||
}
|
||||
function restoreTerms(prev){
|
||||
document.querySelectorAll(".term-body").forEach((el) => {
|
||||
const id = el.dataset.termId || "";
|
||||
const item = prev[id];
|
||||
if (!item || item.follow) el.scrollTop = el.scrollHeight;
|
||||
else el.scrollTop = item.top || 0;
|
||||
el.addEventListener("scroll", saveScroll, { passive: true });
|
||||
});
|
||||
}
|
||||
function saveScroll(){
|
||||
const next = { pageY: window.scrollY, terms: captureTerms() };
|
||||
sessionStorage.setItem(pageKey, JSON.stringify(next));
|
||||
}
|
||||
function setText(id, value){
|
||||
const el = document.getElementById(id);
|
||||
if (el) el.textContent = value == null ? "" : String(value);
|
||||
}
|
||||
function setHtml(id, value){
|
||||
const el = document.getElementById(id);
|
||||
if (el) el.innerHTML = value || "";
|
||||
}
|
||||
function applyFilter(filter){
|
||||
currentFilter = filter || "all";
|
||||
sessionStorage.setItem(pageKey + ":filter", currentFilter);
|
||||
document.querySelectorAll("[data-filter]").forEach((b) => b.classList.toggle("on", b.dataset.filter === currentFilter));
|
||||
document.querySelectorAll(".mc").forEach((card) => {
|
||||
card.hidden = currentFilter !== "all" && card.dataset.status !== currentFilter;
|
||||
});
|
||||
}
|
||||
function applyState(state){
|
||||
if (!state) return;
|
||||
const previousTerms = captureTerms();
|
||||
const previousY = window.scrollY;
|
||||
setText("brand-title", state.title);
|
||||
setText("brand-project", state.project);
|
||||
setText("clock-time", state.clock);
|
||||
setText("clock-date", state.generatedAt);
|
||||
setText("summary-online", `${state.onlineTotal}/${state.deviceTotal} 在线 · ${state.runTotal} RUNS`);
|
||||
setText("summary-counts", `${state.trainingTotal} training · ${state.idleTotal} idle · ${state.errorTotal} error · ${state.offlineTotal} offline`);
|
||||
setText("count-all", state.deviceTotal);
|
||||
setText("count-training", state.trainingTotal);
|
||||
setText("count-idle", state.idleTotal);
|
||||
setText("count-error", state.errorTotal);
|
||||
setText("count-offline", state.offlineTotal);
|
||||
setHtml("heatmap", state.heatCellsHtml);
|
||||
setHtml("device-grid", state.cardsHtml);
|
||||
setHtml("runs-root", state.runsHtml);
|
||||
setText("ticker-generated", `generated ${state.generatedAt}`);
|
||||
restoreTerms(previousTerms);
|
||||
applyFilter(currentFilter);
|
||||
window.scrollTo(0, previousY);
|
||||
}
|
||||
window.addEventListener("autodl-dashboard-state", (event) => applyState(event.detail));
|
||||
function loadState(){
|
||||
if (!stateSrc) return;
|
||||
const script = document.createElement("script");
|
||||
script.src = stateSrc + "?t=" + Date.now();
|
||||
script.async = true;
|
||||
script.onload = () => script.remove();
|
||||
script.onerror = () => script.remove();
|
||||
document.head.appendChild(script);
|
||||
}
|
||||
window.addEventListener("beforeunload", saveScroll);
|
||||
window.addEventListener("pagehide", saveScroll);
|
||||
document.querySelectorAll("[data-filter]").forEach((btn) => {
|
||||
btn.addEventListener("click", () => applyFilter(btn.dataset.filter));
|
||||
});
|
||||
applyState(window.__AUTODL_DASHBOARD_STATE__);
|
||||
const saved = JSON.parse(sessionStorage.getItem(pageKey) || "{}");
|
||||
if (typeof saved.pageY === "number") window.scrollTo(0, saved.pageY);
|
||||
if (refreshSeconds > 0) window.setInterval(loadState, refreshSeconds * 1000);
|
||||
})();
|
||||
EOF
|
||||
}
|
||||
|
||||
render_dashboard_state() {
|
||||
local fleet="$1" state_path="$2" lines="$3" generated_at title output_dir tmp_state cards_tmp runs_tmp heat_tmp
|
||||
output_dir="$(dirname "$state_path")"
|
||||
mkdir -p "$output_dir"
|
||||
generated_at="$(date +%Y-%m-%dT%H:%M:%S%z)"
|
||||
if [ -n "$fleet" ]; then
|
||||
[ -d "$(fleet_dir "$fleet")" ] || die "Fleet not found: $fleet"
|
||||
@@ -1626,39 +1889,80 @@ render_dashboard() {
|
||||
else
|
||||
title="Current Remote"
|
||||
fi
|
||||
|
||||
DASHBOARD_DEVICE_TOTAL=0
|
||||
DASHBOARD_ONLINE_TOTAL=0
|
||||
DASHBOARD_TRAINING_TOTAL=0
|
||||
DASHBOARD_IDLE_TOTAL=0
|
||||
DASHBOARD_ERROR_TOTAL=0
|
||||
DASHBOARD_OFFLINE_TOTAL=0
|
||||
DASHBOARD_RUN_TOTAL=0
|
||||
DASHBOARD_HEAT_CELLS=""
|
||||
|
||||
cards_tmp="$(mktemp "$output_dir/.dashboard-cards.XXXXXX")" || die "mktemp failed"
|
||||
runs_tmp="$(mktemp "$output_dir/.dashboard-runs.XXXXXX")" || { rm -f "$cards_tmp"; die "mktemp failed"; }
|
||||
heat_tmp="$(mktemp "$output_dir/.dashboard-heat.XXXXXX")" || { rm -f "$cards_tmp" "$runs_tmp"; die "mktemp failed"; }
|
||||
dashboard_device_cards "$fleet" "$lines" > "$cards_tmp"
|
||||
dashboard_run_tiles "$fleet" > "$runs_tmp"
|
||||
printf '%s' "$DASHBOARD_HEAT_CELLS" > "$heat_tmp"
|
||||
|
||||
tmp_state="$(mktemp "$output_dir/.dashboard-state.XXXXXX")" || { rm -f "$cards_tmp" "$runs_tmp" "$heat_tmp"; die "mktemp failed"; }
|
||||
{
|
||||
printf '<!doctype html><html><head><meta charset="utf-8"><title>AutoDL Remote Dashboard</title><style>'
|
||||
printf 'body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,sans-serif;margin:24px;background:#f6f7f9;color:#172033}h1{font-size:24px}table{border-collapse:collapse;width:100%%;background:#fff;margin:16px 0;border:1px solid #d9dee7}th,td{text-align:left;padding:9px 10px;border-bottom:1px solid #edf0f5;font-size:14px;vertical-align:top}th{background:#eef2f7}.online{color:#087f5b;font-weight:700}.offline{color:#c92a2a;font-weight:700}.muted{color:#667085;font-size:12px;margin-top:4px}code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px}.summary{color:#475467;margin-bottom:18px}.log-row td{background:#0f172a;color:#dbeafe;padding:0}.log-title{font-size:12px;color:#93c5fd;padding:10px 12px;border-bottom:1px solid #1e293b}.log{margin:0;padding:12px;white-space:pre-wrap;overflow:auto;max-height:360px;font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px;line-height:1.45}.badge{display:inline-block;border:1px solid #d0d5dd;border-radius:999px;padding:2px 8px;font-size:12px;color:#475467;background:#fff}'
|
||||
printf '(function(){const d=(s)=>new TextDecoder().decode(Uint8Array.from(atob(s),c=>c.charCodeAt(0)));'
|
||||
printf 'const state={'
|
||||
printf 'generatedAt:%s,' "$(js_string "$generated_at")"
|
||||
printf 'clock:%s,' "$(js_string "$(date +%H:%M:%S)")"
|
||||
printf 'title:%s,' "$(js_string "$title")"
|
||||
printf 'project:%s,' "$(js_string "$PROJECT_ROOT")"
|
||||
printf 'deviceTotal:%s,onlineTotal:%s,trainingTotal:%s,idleTotal:%s,errorTotal:%s,offlineTotal:%s,runTotal:%s,' \
|
||||
"$DASHBOARD_DEVICE_TOTAL" "$DASHBOARD_ONLINE_TOTAL" "$DASHBOARD_TRAINING_TOTAL" "$DASHBOARD_IDLE_TOTAL" "$DASHBOARD_ERROR_TOTAL" "$DASHBOARD_OFFLINE_TOTAL" "$DASHBOARD_RUN_TOTAL"
|
||||
printf 'heatCellsHtml:d("%s"),' "$(base64_file_compact "$heat_tmp")"
|
||||
printf 'cardsHtml:d("%s"),' "$(base64_file_compact "$cards_tmp")"
|
||||
printf 'runsHtml:d("%s")' "$(base64_file_compact "$runs_tmp")"
|
||||
printf '};window.__AUTODL_DASHBOARD_STATE__=state;window.dispatchEvent(new CustomEvent("autodl-dashboard-state",{detail:state}));})();\n'
|
||||
} > "$tmp_state"
|
||||
mv "$tmp_state" "$state_path"
|
||||
rm -f "$cards_tmp" "$runs_tmp" "$heat_tmp"
|
||||
}
|
||||
|
||||
render_dashboard() {
|
||||
local fleet="$1" output="$2" lines="$3" refresh_seconds="${4:-}" generated_at title output_dir tmp_output state_path state_src
|
||||
output_dir="$(dirname "$output")"
|
||||
mkdir -p "$output_dir"
|
||||
generated_at="$(date +%Y-%m-%dT%H:%M:%S%z)"
|
||||
if [ -n "$fleet" ]; then
|
||||
[ -d "$(fleet_dir "$fleet")" ] || die "Fleet not found: $fleet"
|
||||
title="Fleet $fleet"
|
||||
else
|
||||
title="Current Remote"
|
||||
fi
|
||||
state_path="$(dashboard_state_path "$output")"
|
||||
state_src="$(basename "$state_path")"
|
||||
render_dashboard_state "$fleet" "$state_path" "$lines"
|
||||
|
||||
tmp_output="$(mktemp "$output_dir/.dashboard.XXXXXX")" || die "mktemp failed"
|
||||
{
|
||||
printf '<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>AutoDL Remote Dashboard</title><style>'
|
||||
dashboard_css
|
||||
printf '</style>'
|
||||
printf '</head><body><div class="app"><div class="scanlines"></div><div class="vignette"></div>'
|
||||
printf '<header class="topbar"><div class="topbar-l"><div class="brand"><span class="brand-mark">◇</span><span class="brand-name">AUTODL</span><span class="brand-sub">MONITOR</span><span class="brand-ver">v%s</span></div><div class="brand-tagline">— <span id="brand-title">%s</span> · <span id="brand-project">%s</span> —</div></div>' "$(html_escape "$PROGRAM_VERSION")" "$(html_escape "$title")" "$(html_escape "$PROJECT_ROOT")"
|
||||
printf '<div class="topbar-c"><div class="agg-block"><div class="agg-block-h"><span>TOTAL UTIL</span><span id="summary-online">loading</span></div><div id="heatmap" class="heatmap"></div><div id="summary-counts" class="agg-block-f">loading</div></div></div>'
|
||||
printf '<div class="topbar-r"><div class="clock"><div id="clock-time" class="clock-t">%s</div><div id="clock-date" class="clock-d">%s</div></div><div class="refresh"><span class="dim">REFRESH</span><div class="seg">' "$(date +%H:%M:%S)" "$(html_escape "$generated_at")"
|
||||
if [ -n "$refresh_seconds" ]; then
|
||||
printf '<meta http-equiv="refresh" content="%s">' "$(html_escape "$refresh_seconds")"
|
||||
fi
|
||||
printf '</head><body>'
|
||||
printf '<h1>AutoDL Remote Dashboard</h1><div class="summary">%s · generated %s · project %s' "$(html_escape "$title")" "$(html_escape "$generated_at")" "$(html_escape "$PROJECT_ROOT")"
|
||||
if [ -n "$refresh_seconds" ]; then
|
||||
printf ' · <span class="badge">refresh every %ss</span>' "$(html_escape "$refresh_seconds")"
|
||||
fi
|
||||
printf '</div>'
|
||||
printf '<h2>Devices</h2><table><thead><tr><th>Device</th><th>Online</th><th>Account</th><th>GPU</th><th>Disk</th><th>Current Run</th><th>Tags</th></tr></thead><tbody>'
|
||||
if [ -n "$fleet" ]; then
|
||||
local device_file device account remote tags
|
||||
for device_file in "$(fleet_dir "$fleet")"/*.conf; do
|
||||
[ -f "$device_file" ] || continue
|
||||
device="$(device_get "$device_file" NAME)"
|
||||
account="$(device_get "$device_file" ACCOUNT)"
|
||||
remote="$(device_get "$device_file" REMOTE_ROOT)"
|
||||
tags="$(device_get "$device_file" TAGS)"
|
||||
dashboard_device_row "$device" "$account" "$remote" "$tags"
|
||||
done
|
||||
printf '<button class="seg-btn on">%ss</button>' "$(html_escape "$refresh_seconds")"
|
||||
else
|
||||
dashboard_device_row "current" "$PROJECT_ACCOUNT" "$REMOTE_ROOT" "project"
|
||||
printf '<button class="seg-btn on">manual</button>'
|
||||
fi
|
||||
printf '</tbody></table>'
|
||||
printf '<h2>Runs</h2><table><thead><tr><th>Run</th><th>Account</th><th>Status</th><th>Model</th><th>Stage</th><th>Started</th><th>Tags</th><th>Log / Notes</th></tr></thead><tbody>'
|
||||
dashboard_run_rows "$fleet" "$lines"
|
||||
printf '</tbody></table>'
|
||||
printf '</body></html>\n'
|
||||
} > "$output"
|
||||
printf '</div></div></div><div class="filters"><button class="flt on" data-filter="all"><span>全部</span><span id="count-all" class="flt-c">0</span></button><button class="flt" data-filter="training"><span>训练中</span><span id="count-training" class="flt-c">0</span></button><button class="flt" data-filter="idle"><span>空闲</span><span id="count-idle" class="flt-c">0</span></button><button class="flt" data-filter="error"><span>异常</span><span id="count-error" class="flt-c">0</span></button><button class="flt" data-filter="offline"><span>离线</span><span id="count-offline" class="flt-c">0</span></button></div></header>'
|
||||
printf '<main id="device-grid" class="grid"></main><div id="runs-root"></div>'
|
||||
printf '<div class="ticker"><span class="ticker-tag">EVT</span><div class="ticker-row"><span id="ticker-generated">generated %s</span><span>read-only dashboard</span><span>state updates without full page reload</span></div></div>' "$(html_escape "$generated_at")"
|
||||
printf '<script>window.__AUTODL_REFRESH_SECONDS__=%s;window.__AUTODL_STATE_SRC__=%s;</script>' "${refresh_seconds:-0}" "$(js_string "$state_src")"
|
||||
printf '<script src="%s"></script><script>' "$(html_escape "$state_src")"
|
||||
dashboard_js
|
||||
printf '</script></div></body></html>\n'
|
||||
} > "$tmp_output"
|
||||
mv "$tmp_output" "$output"
|
||||
}
|
||||
|
||||
cmd_dashboard() {
|
||||
@@ -1684,14 +1988,16 @@ cmd_dashboard() {
|
||||
fi
|
||||
[ -z "$output" ] && output="$PROJECT_ROOT/$HISTORY_DIR/dashboard.html"
|
||||
if [ -n "$watch_seconds" ]; then
|
||||
render_dashboard "$fleet" "$output" "$lines" "$watch_seconds"
|
||||
info "Dashboard written: $output"
|
||||
if [ "$open_file" = "true" ] && [ "$opened" = "false" ]; then
|
||||
command -v open >/dev/null 2>&1 && open "$output" || info "open command not available."
|
||||
opened=true
|
||||
fi
|
||||
while :; do
|
||||
render_dashboard "$fleet" "$output" "$lines" "$watch_seconds"
|
||||
info "Dashboard written: $output"
|
||||
if [ "$open_file" = "true" ] && [ "$opened" = "false" ]; then
|
||||
command -v open >/dev/null 2>&1 && open "$output" || info "open command not available."
|
||||
opened=true
|
||||
fi
|
||||
sleep "$watch_seconds"
|
||||
render_dashboard_state "$fleet" "$(dashboard_state_path "$output")" "$lines"
|
||||
info "Dashboard state updated: $(dashboard_state_path "$output")"
|
||||
done
|
||||
fi
|
||||
render_dashboard "$fleet" "$output" "$lines"
|
||||
|
||||
@@ -17,6 +17,8 @@ Use this skill when the user wants Codex to work locally while controlling AutoD
|
||||
- Do not decide that local or remote is the source of truth globally.
|
||||
- Do not use `local-main`, `remote-main`, manifests, sparse-cache modes, or automatic code ownership rules.
|
||||
- Codex decides the smallest useful action each time: inspect, upload, download, run, or tail logs.
|
||||
- Reusable project assets must be created or edited locally first, then uploaded with `put`, `sync-up`, or `put-run` before remote execution. This includes source code, notebooks, YAML/JSON/TOML configs, prompts, templates, and files needed to reproduce an experiment. Do not create or modify these durable files directly on the remote machine with heredocs, `cat > file`, editors, or shell redirection, unless the user explicitly asks for a remote-only edit.
|
||||
- Shell commands and shell launch wrappers are normally runtime control, not durable project assets. They may be generated or executed remotely without local persistence for startup glue, environment checks, quoting-heavy commands, process supervision, and one-off experiment launches. Only save shell scripts locally when the user asks to keep them or they become a named project entrypoint.
|
||||
- Do not download LLM weights, checkpoints, datasets, or large outputs unless the user explicitly asks.
|
||||
- Prefer reading remote files/logs in place with `cat`, `tail`, `ls`, and `tree`.
|
||||
- Use detached job commands for long training runs instead of manually guessing process state.
|
||||
@@ -89,6 +91,7 @@ autodl-remote shutdown
|
||||
|
||||
- Before editing remote-existing code, inspect with `tree`, `ls`, `cat`, or `get` only the needed files.
|
||||
- Before running remote code that depends on local edits, explicitly upload the edited files or directories with `put` or `sync-up`.
|
||||
- Do not use `exec --stdin`, heredocs, `cat >`, remote editors, or remote shell redirection to create reusable project assets on the remote host. Keep durable code and configuration local-first so git and the user's local workspace remain the record. This restriction does not apply to shell commands or shell wrappers used to start, check, or supervise runs.
|
||||
- After remote execution, inspect logs remotely first; pull only small result files when useful.
|
||||
- If both local and remote have similar files, do not assume one should overwrite the other. Compare or inspect first, then choose `put` or `get`.
|
||||
- Use `exec --detach` for long training jobs and then `tail` the log path.
|
||||
|
||||
Reference in new issue
Block a user