feat(cpu): global dynamic CPU limit rule
- admin panel gains a global dynamic CPU limit card (default off, prefilled 10/60/0.5/2/30): a container that keeps using over y% of its own quota for x consecutive minutes is capped to z cores for n hours m minutes, then its normal quota is restored automatically - reuse lx.SetCPU (live instance-config PATCH) and the per-minute resource_samples.cpu_pct_x10 (already relative to the container's quota); the new EnforceCPULimits pass runs in sampleResourceLoop, and the rule plus the active-limit state live in DB settings so a vps.service restart keeps every countdown and still restores on expiry - user panel shows the capped core count and a frontend countdown while a limit is active; containers whose own quota is <= z are skipped - cover the rule validation, streak detection, active-state cleanup and the user-panel badge in unit tests Co-authored-by: CommandCodeBot <noreply@commandcode.ai>
This commit is contained in:
1 parent
3df47aece0
commit
0e4528ad85
15 files changed
+812
-25
No files matched your search
@@ -77,6 +77,7 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("/power", s.requireAuth(s.requirePost(s.handlePower)))
|
||||
mux.HandleFunc("/reset-panel-pass", s.requireAuth(s.requirePost(s.handleResetPanelPass)))
|
||||
mux.HandleFunc("/admin-pass", s.requireAuth(s.requirePost(s.handleAdminPass)))
|
||||
mux.HandleFunc("/cpu-limit", s.requireAuth(s.requirePost(s.handleCPULimitRule)))
|
||||
mux.HandleFunc("/keys", s.requireAuth(s.requirePost(s.handleAdminKeys)))
|
||||
mux.HandleFunc("/login-as", s.requireAuth(s.requirePost(s.handleLoginAs)))
|
||||
mux.HandleFunc("/flash", s.requireAuth(s.requirePost(s.handleFlash)))
|
||||
|
||||
@@ -2,7 +2,9 @@ package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -48,6 +50,23 @@ type pageData struct {
|
||||
PoolTotal int
|
||||
// AdminKeys is the operator's own SSH-key store (management panel only).
|
||||
AdminKeys []sshKeyRow
|
||||
// Global dynamic CPU limit rule, prefilled into the overview form. The
|
||||
// Active list is the containers currently capped by it.
|
||||
CPULimitEnabled bool
|
||||
CPULimitMinutes int
|
||||
CPULimitPercent int
|
||||
CPULimitCores string
|
||||
CPULimitHours int
|
||||
CPULimitDurMin int
|
||||
CPULimitActive []cpuLimitRow
|
||||
}
|
||||
|
||||
// cpuLimitRow is one container currently under the dynamic CPU limit, shown in
|
||||
// the rule card (name, the cap, and when it expires).
|
||||
type cpuLimitRow struct {
|
||||
Name string
|
||||
Cores string
|
||||
Expires string
|
||||
}
|
||||
|
||||
// sshKeyRow is one public key shown in the admin SSH-key management panel.
|
||||
@@ -145,9 +164,49 @@ func (s *Server) buildPageData(msg, errMsg string) pageData {
|
||||
d.AdminKeys = append(d.AdminKeys, sshKeyRow{ID: k.ID, Name: k.Name, Key: k.Key, Active: k.Active})
|
||||
}
|
||||
}
|
||||
if rule, err := s.mgr.CPULimitRule(); err == nil {
|
||||
d.CPULimitEnabled = rule.Enabled
|
||||
d.CPULimitMinutes = rule.WindowMinutes
|
||||
d.CPULimitPercent = rule.Percent
|
||||
d.CPULimitCores = mgr.FormatCPU(rule.CoresX10)
|
||||
d.CPULimitHours = rule.DurationSeconds / 3600
|
||||
d.CPULimitDurMin = (rule.DurationSeconds % 3600) / 60
|
||||
}
|
||||
nowUnix := time.Now().Unix()
|
||||
for name, st := range s.mgr.CPULimits() {
|
||||
if st.Until <= nowUnix {
|
||||
continue
|
||||
}
|
||||
d.CPULimitActive = append(d.CPULimitActive, cpuLimitRow{
|
||||
Name: name,
|
||||
Cores: mgr.FormatCPU(st.CoresX10),
|
||||
Expires: fmtCPURemaining(time.Unix(st.Until, 0)),
|
||||
})
|
||||
}
|
||||
sort.Slice(d.CPULimitActive, func(i, j int) bool { return d.CPULimitActive[i].Name < d.CPULimitActive[j].Name })
|
||||
return d
|
||||
}
|
||||
|
||||
// fmtCPURemaining renders the time left on a dynamic CPU limit compactly
|
||||
// ("2h30m", "5m12s", "40s").
|
||||
func fmtCPURemaining(until time.Time) string {
|
||||
d := time.Until(until).Truncate(time.Second)
|
||||
if d < 0 {
|
||||
d = 0
|
||||
}
|
||||
h := int(d.Hours())
|
||||
m := int(d.Minutes()) % 60
|
||||
s := int(d.Seconds()) % 60
|
||||
switch {
|
||||
case h > 0:
|
||||
return fmt.Sprintf("%dh%dm", h, m)
|
||||
case m > 0:
|
||||
return fmt.Sprintf("%dm%ds", m, s)
|
||||
default:
|
||||
return fmt.Sprintf("%ds", s)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) loadUsers(d *pageData) {
|
||||
statuses, err := s.mgr.BatchUsers()
|
||||
if err != nil {
|
||||
@@ -592,6 +651,64 @@ func (s *Server) handleAdminPass(w http.ResponseWriter, r *http.Request) {
|
||||
s.redirect(w, r, s.p(""), s.t(r, "admin_pass_changed"))
|
||||
}
|
||||
|
||||
// handleCPULimitRule saves the global dynamic CPU limit rule and applies it
|
||||
// immediately: a disabled rule restores every capped container right away
|
||||
// instead of waiting for the next 60s pass.
|
||||
func (s *Server) handleCPULimitRule(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
enabled := r.FormValue("enabled") != ""
|
||||
minutes, err := strconv.Atoi(strings.TrimSpace(r.FormValue("window_minutes")))
|
||||
if err != nil || minutes < 1 {
|
||||
s.redirect(w, r, s.p(""), "error: "+s.t(r, "err_cpu_limit_window"))
|
||||
return
|
||||
}
|
||||
percent, err := strconv.Atoi(strings.TrimSpace(r.FormValue("percent")))
|
||||
if err != nil || percent < 1 || percent > 100 {
|
||||
s.redirect(w, r, s.p(""), "error: "+s.t(r, "err_cpu_limit_percent"))
|
||||
return
|
||||
}
|
||||
cores, err := mgr.ParseLimitCores(r.FormValue("cores"))
|
||||
if err != nil {
|
||||
s.redirect(w, r, s.p(""), "error: "+s.t(r, "err_cpu_limit_cores"))
|
||||
return
|
||||
}
|
||||
hours, err := strconv.Atoi(strings.TrimSpace(r.FormValue("hours")))
|
||||
if err != nil || hours < 0 {
|
||||
s.redirect(w, r, s.p(""), "error: "+s.t(r, "err_cpu_limit_duration"))
|
||||
return
|
||||
}
|
||||
durMin, err := strconv.Atoi(strings.TrimSpace(r.FormValue("dur_minutes")))
|
||||
if err != nil || durMin < 0 || durMin > 59 {
|
||||
s.redirect(w, r, s.p(""), "error: "+s.t(r, "err_cpu_limit_duration"))
|
||||
return
|
||||
}
|
||||
duration := hours*3600 + durMin*60
|
||||
if enabled && duration <= 0 {
|
||||
s.redirect(w, r, s.p(""), "error: "+s.t(r, "err_cpu_limit_duration"))
|
||||
return
|
||||
}
|
||||
rule := mgr.CPULimitRule{
|
||||
Enabled: enabled,
|
||||
WindowMinutes: minutes,
|
||||
Percent: percent,
|
||||
CoresX10: cores,
|
||||
DurationSeconds: duration,
|
||||
}
|
||||
if err := s.mgr.SetCPULimitRule(rule); err != nil {
|
||||
s.redirect(w, r, s.p(""), "error: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.mgr.EnforceCPULimits(); err != nil {
|
||||
s.redirect(w, r, s.p(""), "error: "+err.Error())
|
||||
return
|
||||
}
|
||||
_ = s.db.AddAuditLog("000", "cpu_limit.update")
|
||||
s.redirect(w, r, s.p(""), s.t(r, "cpu_limit_saved"))
|
||||
}
|
||||
|
||||
// handleLoginAs ("log in as user" / impersonation) creates a user-panel
|
||||
// session for the given username and hands the browser its cookie, dropping the
|
||||
// operator straight into that user's panel — regardless of the user's password.
|
||||
|
||||
+28
-23
@@ -79,29 +79,34 @@ func (s *Server) t(r *http.Request, key string, args ...any) string {
|
||||
// stable identifiers; args are %-formatted into the chosen language's template.
|
||||
func tr(l, key string, args ...any) string {
|
||||
m := map[string][2]string{ // key -> [zh, en]
|
||||
"err_bad_login": {"管理员密码错误", "invalid admin password"},
|
||||
"err_too_many": {"尝试过于频繁,请 1 分钟后再试", "Too many attempts, please wait 1 minute"},
|
||||
"err_not_configured": {"管理员尚未初始化 — 请运行 `vps admin-passwd`", "admin not configured yet — run `vps admin-passwd`"},
|
||||
"err_pass_mismatch": {"两次输入的密码不一致", "The two passwords do not match"},
|
||||
"err_pass_short": {"密码长度不得低于 10 位", "password must be at least 10 characters"},
|
||||
"err_pass_weak": {"密码需同时包含大写字母、小写字母和数字", "password must contain uppercase letters, lowercase letters and digits"},
|
||||
"user_created": {"用户已创建:\n%[1]v", "user created:\n%[1]v"},
|
||||
"user_deleted": {"用户 %[1]v 已删除", "user %[1]v deleted"},
|
||||
"quota_updated": {"已更新 %[1]v 的配额", "quotas updated for %[1]v"},
|
||||
"bandwidth_reset": {"已重置 %[1]v 的流量统计(若曾限速则已解除)", "bandwidth reset for %[1]v (throttle lifted if active)"},
|
||||
"power_ok": {"%[1]v %[2]v 成功", "%[2]v %[1]v ok"},
|
||||
"new_panel_password": {"用户 %[1]v 的面板密码已重置:\n%[2]v\n面板:%[3]v", "%[1]v panel password reset:\n%[2]v\npanel: %[3]v"},
|
||||
"admin_pass_changed": {"管理员密码已修改", "admin password changed"},
|
||||
"err_invalid_disk": {"磁盘必须是整数 GiB", "disk must be an integer (GiB)"},
|
||||
"domain_deleted": {"域名 %[1]v 已删除", "domain %[1]v deleted"},
|
||||
"domains_updated": {"域名设置已保存", "domain settings saved"},
|
||||
"blocked_saved": {"封禁域名列表已保存", "blocked-domains list saved"},
|
||||
"blocked_skipped_lines": {"第 %[1]v 行不是有效域名,已跳过;其余已保存", "line(s) %[1]v are not valid domains and were skipped; the rest were saved"},
|
||||
"pool_added": {"已添加 %[1]v 个 IPv6 地址到池", "added %[1]v IPv6 address(es) to the pool"},
|
||||
"pool_removed": {"已从池中移除 %[1]v", "removed %[1]v from the pool"},
|
||||
"color_updated": {"已设置 %[1]v 的专属颜色", "color set for %[1]v"},
|
||||
"color_reset": {"已清除 %[1]v 的专属颜色", "color cleared for %[1]v"},
|
||||
"err_invalid_color": {"颜色不在可选范围内", "color is not in the allowed palette"},
|
||||
"err_bad_login": {"管理员密码错误", "invalid admin password"},
|
||||
"err_too_many": {"尝试过于频繁,请 1 分钟后再试", "Too many attempts, please wait 1 minute"},
|
||||
"err_not_configured": {"管理员尚未初始化 — 请运行 `vps admin-passwd`", "admin not configured yet — run `vps admin-passwd`"},
|
||||
"err_pass_mismatch": {"两次输入的密码不一致", "The two passwords do not match"},
|
||||
"err_pass_short": {"密码长度不得低于 10 位", "password must be at least 10 characters"},
|
||||
"err_pass_weak": {"密码需同时包含大写字母、小写字母和数字", "password must contain uppercase letters, lowercase letters and digits"},
|
||||
"user_created": {"用户已创建:\n%[1]v", "user created:\n%[1]v"},
|
||||
"user_deleted": {"用户 %[1]v 已删除", "user %[1]v deleted"},
|
||||
"quota_updated": {"已更新 %[1]v 的配额", "quotas updated for %[1]v"},
|
||||
"bandwidth_reset": {"已重置 %[1]v 的流量统计(若曾限速则已解除)", "bandwidth reset for %[1]v (throttle lifted if active)"},
|
||||
"power_ok": {"%[1]v %[2]v 成功", "%[2]v %[1]v ok"},
|
||||
"new_panel_password": {"用户 %[1]v 的面板密码已重置:\n%[2]v\n面板:%[3]v", "%[1]v panel password reset:\n%[2]v\npanel: %[3]v"},
|
||||
"admin_pass_changed": {"管理员密码已修改", "admin password changed"},
|
||||
"err_invalid_disk": {"磁盘必须是整数 GiB", "disk must be an integer (GiB)"},
|
||||
"domain_deleted": {"域名 %[1]v 已删除", "domain %[1]v deleted"},
|
||||
"domains_updated": {"域名设置已保存", "domain settings saved"},
|
||||
"blocked_saved": {"封禁域名列表已保存", "blocked-domains list saved"},
|
||||
"blocked_skipped_lines": {"第 %[1]v 行不是有效域名,已跳过;其余已保存", "line(s) %[1]v are not valid domains and were skipped; the rest were saved"},
|
||||
"pool_added": {"已添加 %[1]v 个 IPv6 地址到池", "added %[1]v IPv6 address(es) to the pool"},
|
||||
"pool_removed": {"已从池中移除 %[1]v", "removed %[1]v from the pool"},
|
||||
"color_updated": {"已设置 %[1]v 的专属颜色", "color set for %[1]v"},
|
||||
"color_reset": {"已清除 %[1]v 的专属颜色", "color cleared for %[1]v"},
|
||||
"err_invalid_color": {"颜色不在可选范围内", "color is not in the allowed palette"},
|
||||
"cpu_limit_saved": {"CPU 动态限制规则已保存", "dynamic CPU limit rule saved"},
|
||||
"err_cpu_limit_window": {"连续时长必须是 ≥1 的整数分钟", "window must be an integer >= 1 minute"},
|
||||
"err_cpu_limit_percent": {"百分比必须是 1..100 的整数", "percent must be an integer between 1 and 100"},
|
||||
"err_cpu_limit_cores": {"限制核心数必须是 0.1..1.0(步长 0.1)", "cores must be a one-decimal value in 0.1..1.0"},
|
||||
"err_cpu_limit_duration": {"时长必须有效:小时 ≥0,分钟 0..59,且启用时总时长 >0", "duration must be valid: hours >= 0, minutes 0..59, and > 0 when enabled"},
|
||||
}
|
||||
pair, ok := m[key]
|
||||
if !ok {
|
||||
|
||||
@@ -98,6 +98,7 @@ form.inline[hidden] { display: none; }
|
||||
.field small.hint-bad { color: #b91c1c; }
|
||||
@media (prefers-color-scheme: dark) { .field small.hint-bad { color: #f87171; } }
|
||||
#adminPassForm .pwcheck { grid-column: 1 / -1; margin: -2px 0 0; }
|
||||
#cpuLimitForm .cpu-limit-off input[type="number"] { opacity: .5; }
|
||||
.pwcheck { list-style: none; margin: 6px 0 0; padding: 0; font-size: 12px; color: #6b7280; }
|
||||
@media (prefers-color-scheme: dark) { .pwcheck { color: #9aa0a8; } }
|
||||
.pwcheck li { display: inline-block; margin: 2px 14px 2px 0; }
|
||||
@@ -306,6 +307,35 @@ a.copy.copied { border-color: #86efac; background: #dcfce7; color: #15803d; }
|
||||
<button type="submit" class="primary neutral" id="adminPassBtn">{{if eq .Lang "zh"}}修改{{else}}Change{{end}}</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>{{if eq .Lang "zh"}}CPU 动态限制规则(全局){{else}}Dynamic CPU limit rule (global){{end}}</h2>
|
||||
<form action="{{.Prefix}}/cpu-limit" method="post" class="form-grid" id="cpuLimitForm">
|
||||
<div class="field" style="grid-column:1/-1">
|
||||
<label style="display:flex;align-items:center;gap:8px;font-weight:600">
|
||||
<input type="checkbox" name="enabled" id="cpuLimitEnabled" {{if .CPULimitEnabled}}checked{{end}}>
|
||||
{{if eq .Lang "zh"}}启用限制(默认:不限制){{else}}Enable limit (default: no limit){{end}}
|
||||
</label>
|
||||
</div>
|
||||
<div class="field"><label>{{if eq .Lang "zh"}}连续 x 分钟{{else}}Consecutive x minutes{{end}}</label><input name="window_minutes" id="cpuLimitWindow" type="number" min="1" step="1" value="{{.CPULimitMinutes}}" required></div>
|
||||
<div class="field"><label>{{if eq .Lang "zh"}}占用配额超过 y%{{else}}Over quota y%{{end}}</label><input name="percent" id="cpuLimitPercent" type="number" min="1" max="100" step="1" value="{{.CPULimitPercent}}" required title="{{if eq .Lang "zh"}}相对于容器自身配额;如 4 核配额下 60% = 单分钟持续超过 2.4 核{{else}}relative to the container's own quota; e.g. 60% of a 4-core quota = sustained >2.4 cores for a minute{{end}}"></div>
|
||||
<div class="field"><label>{{if eq .Lang "zh"}}限制到 z 核{{else}}Limit to z cores{{end}}</label><input name="cores" id="cpuLimitCores" type="number" min="0.1" max="1" step="0.1" value="{{.CPULimitCores}}" required title="{{if eq .Lang "zh"}}0.1~1.0(步长 0.1),单核的时间片占比{{else}}0.1..1.0 (steps of 0.1), a time slice of one core{{end}}"></div>
|
||||
<div class="field"><label>{{if eq .Lang "zh"}}持续 n 小时{{else}}For n hours{{end}}</label><input name="hours" id="cpuLimitHours" type="number" min="0" step="1" value="{{.CPULimitHours}}" required></div>
|
||||
<div class="field"><label>{{if eq .Lang "zh"}}m 分钟{{else}}m minutes{{end}}</label><input name="dur_minutes" id="cpuLimitDurMin" type="number" min="0" max="59" step="1" value="{{.CPULimitDurMin}}" required></div>
|
||||
<button type="submit" class="primary neutral" id="cpuLimitBtn">{{if eq .Lang "zh"}}保存{{else}}Save{{end}}</button>
|
||||
</form>
|
||||
<small style="color:#9aa0a8;display:block;margin-top:8px">{{if eq .Lang "zh"}}容器连续 x 分钟占用其配额超过 y% 时,限制到 z 核,持续 n 小时 m 分钟后自动恢复原配额。配额本身不高于 z 的容器会被跳过。{{else}}When a container stays above y% of its quota for x minutes, it is capped to z cores for n hours m minutes, then its quota is restored. Containers whose own quota is at or below z are skipped.{{end}}</small>
|
||||
<div style="margin-top:10px">
|
||||
{{if .CPULimitActive}}
|
||||
<div style="font-size:12px;color:#6b7280;margin-bottom:6px">{{if eq .Lang "zh"}}当前被限制容器{{else}}Currently limited containers{{end}}({{len .CPULimitActive}})</div>
|
||||
<table><thead><tr><th>{{if eq .Lang "zh"}}容器{{else}}Container{{end}}</th><th>{{if eq .Lang "zh"}}限制到{{else}}Limited to{{end}}</th><th>{{if eq .Lang "zh"}}剩余{{else}}Remaining{{end}}</th></tr></thead><tbody>
|
||||
{{range .CPULimitActive}}<tr><td>{{.Name}}</td><td>{{.Cores}} {{if eq $.Lang "zh"}}核{{else}}core(s){{end}}</td><td>{{.Expires}}</td></tr>{{end}}
|
||||
</tbody></table>
|
||||
{{else}}
|
||||
<small style="color:#9aa0a8">{{if eq .Lang "zh"}}当前没有容器被限制{{else}}No container is currently limited{{end}}</small>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<!-- modal -->
|
||||
@@ -668,6 +698,22 @@ var child = low.match(/^([a-z][a-z0-9-]*[a-z])-([0-9]+)$/);
|
||||
btn.disabled = true; btn.classList.add('busy');
|
||||
});
|
||||
})();
|
||||
// Dynamic CPU limit rule: dim the parameters while the rule is disabled (they
|
||||
// stay enabled so their values are always submitted), and show a busy state on
|
||||
// save.
|
||||
(function(){
|
||||
var f = document.getElementById('cpuLimitForm');
|
||||
if (!f) return;
|
||||
var en = document.getElementById('cpuLimitEnabled');
|
||||
function sync(){ f.classList.toggle('cpu-limit-off', !(en && en.checked)); }
|
||||
if (en) en.addEventListener('change', sync);
|
||||
sync();
|
||||
var btn = document.getElementById('cpuLimitBtn');
|
||||
f.addEventListener('submit', function(){
|
||||
if (!f.checkValidity()) return;
|
||||
if (btn) { btn.disabled = true; btn.classList.add('busy'); }
|
||||
});
|
||||
})();
|
||||
// Quota update can take 10s+ (live LXD config changes); show a spinner on the
|
||||
// Save button while the request is in flight so the modal does not look dead.
|
||||
(function(){
|
||||
|
||||
@@ -186,6 +186,37 @@ func (d *DB) AverageCPU(since int64) (map[int64]float64, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RecentResourceSamples returns every user's samples at or after the given
|
||||
// minute, ordered by (user_id, sample_minute). One query feeds the dynamic CPU
|
||||
// limit check, so it can reconstruct each container's trailing run of minutes
|
||||
// over the quota without a per-user round trip.
|
||||
func (d *DB) RecentResourceSamples(since int64) ([]ResourceSample, error) {
|
||||
rows, err := d.sql.Query(`
|
||||
SELECT user_id, sample_minute, state, boot_time,
|
||||
cpu_seconds_ns, cpu_pct_x10, memory_mib, processes,
|
||||
disk_used_mib, rx_bytes_total, tx_bytes_total,
|
||||
disk_read_bytes_total, disk_write_bytes_total
|
||||
FROM resource_samples
|
||||
WHERE sample_minute >= ?
|
||||
ORDER BY user_id, sample_minute`, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ResourceSample
|
||||
for rows.Next() {
|
||||
var s ResourceSample
|
||||
if err := rows.Scan(&s.UserID, &s.SampleMinute, &s.State, &s.BootTime,
|
||||
&s.CPUSecondsNS, &s.CPUPercentX10, &s.MemoryMiB, &s.Processes,
|
||||
&s.DiskUsedMiB, &s.RXBytesTotal, &s.TXBytesTotal,
|
||||
&s.DiskReadBytes, &s.DiskWrittenBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ResourceHistory returns samples for a future resource chart.
|
||||
func (d *DB) ResourceHistory(userID, since int64) ([]ResourceSample, error) {
|
||||
rows, err := d.sql.Query(`
|
||||
|
||||
@@ -49,3 +49,44 @@ func TestRecordResourceSamplesAndRetention(t *testing.T) {
|
||||
t.Fatalf("history = %+v, err=%v", history, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecentResourceSamples(t *testing.T) {
|
||||
d, err := Open(t.TempDir() + "/recent.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
a, err := d.CreateUser("alice", "h", "10.115.0.2", 1, 30001, 10000, 40, 1024, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := d.CreateUser("bob", "h", "10.115.0.3", 2, 30002, 10100, 40, 1024, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sample := func(id, minute, cpu int64) ResourceSample {
|
||||
return ResourceSample{UserID: id, SampleMinute: minute, State: 1,
|
||||
BootTime: 100, CPUSecondsNS: minute * 1e9, CPUPercentX10: cpu}
|
||||
}
|
||||
// alice has samples at 0/60/120, bob only at 120; 0 is outside the window.
|
||||
rows := []ResourceSample{sample(a.ID, 0, 100), sample(a.ID, 60, 200), sample(a.ID, 120, 300), sample(b.ID, 120, 400)}
|
||||
if err := d.RecordResourceSamples(rows, nil, "2026-08", -60); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := d.RecentResourceSamples(60)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("RecentResourceSamples returned %d rows, want 3: %+v", len(got), got)
|
||||
}
|
||||
// Ordered by (user_id, sample_minute): alice's 60/120 then bob's 120.
|
||||
if got[0].UserID != a.ID || got[0].SampleMinute != 60 ||
|
||||
got[1].UserID != a.ID || got[1].SampleMinute != 120 ||
|
||||
got[2].UserID != b.ID || got[2].SampleMinute != 120 {
|
||||
t.Fatalf("unexpected order: %+v", got)
|
||||
}
|
||||
if got[2].CPUPercentX10 != 400 {
|
||||
t.Fatalf("bob cpu pct = %d, want 400", got[2].CPUPercentX10)
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,15 @@ const (
|
||||
// newline-separated normalized domains. A blocked domain and all its
|
||||
// subdomains are refused by AddDomain (admin-managed via the web UI).
|
||||
SettingBlockedDomains = "blocked_domains"
|
||||
|
||||
// SettingCPULimitRule is the admin-managed global dynamic CPU limit rule
|
||||
// (JSON), applied to every container. Absent = the rule is disabled.
|
||||
SettingCPULimitRule = "cpu_limit_rule"
|
||||
|
||||
// SettingCPULimitActive is the JSON map of containers currently under a
|
||||
// dynamic CPU limit (name -> {until, cores_x10}). Persisted so the limit
|
||||
// survives a panel restart and the countdown stays accurate.
|
||||
SettingCPULimitActive = "cpu_limit_active"
|
||||
)
|
||||
|
||||
// GetSetting returns a settings value; ok is false when the key is absent.
|
||||
|
||||
@@ -63,3 +63,23 @@ func FormatCPU(tenths int) string {
|
||||
}
|
||||
return "0." + strconv.Itoa(tenths)
|
||||
}
|
||||
|
||||
// ParseLimitCores parses the dynamic CPU limit target z into tenths of a core.
|
||||
// Unlike ParseCPU it is capped at one core: z is a time-slice limit, so it
|
||||
// accepts any one-decimal value in 0.1..1.0 (e.g. "0.1", "0.5", "1", "1.0"),
|
||||
// yielding 1..10 tenths.
|
||||
func ParseLimitCores(s string) (int, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, errors.New("cores: empty input")
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, errors.New("cores must be a value in 0.1..1.0 (one-decimal steps)")
|
||||
}
|
||||
t := int(f*10 + 0.5)
|
||||
if t < 1 || t > 10 || f*10-float64(t) > 1e-9 || float64(t)-f*10 > 1e-9 {
|
||||
return 0, errors.New("cores must be a value in 0.1..1.0 (one-decimal steps)")
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"vpsmgr/internal/db"
|
||||
)
|
||||
|
||||
// CPULimitRule is the global dynamic CPU limit rule managed from the admin
|
||||
// panel. It applies to every container: when a container keeps its CPU usage
|
||||
// above Percent of its own quota for WindowMinutes in a row, it is capped to
|
||||
// CoresX10 tenths of a core (the same time-slice semantics as a fractional
|
||||
// quota) for DurationSeconds, then its normal quota is restored.
|
||||
type CPULimitRule struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
WindowMinutes int `json:"window_minutes"` // x: consecutive minutes over the threshold
|
||||
Percent int `json:"percent"` // y: percent of the container's own quota, 1..100
|
||||
CoresX10 int `json:"cores_x10"` // z*10: limit target, 1..10 tenths (0.1..1.0 core)
|
||||
DurationSeconds int `json:"duration_seconds"` // how long the limit lasts once applied
|
||||
}
|
||||
|
||||
// CPULimitState is the persisted state of one container's active dynamic limit.
|
||||
type CPULimitState struct {
|
||||
Until int64 `json:"until"` // unix seconds the limit expires
|
||||
CoresX10 int `json:"cores_x10"` // the cap that was applied
|
||||
}
|
||||
|
||||
// DefaultCPULimitRule is the prefilled rule shown in the admin panel before
|
||||
// anything is saved. It is disabled by default ("no limit").
|
||||
func DefaultCPULimitRule() CPULimitRule {
|
||||
return CPULimitRule{
|
||||
Enabled: false,
|
||||
WindowMinutes: 10,
|
||||
Percent: 60,
|
||||
CoresX10: 5,
|
||||
DurationSeconds: 2*3600 + 30*60, // 2h30m
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateCPULimitRule rejects out-of-range rule parameters before they are
|
||||
// persisted, so the enforcement loop only ever sees sane values.
|
||||
func ValidateCPULimitRule(r CPULimitRule) error {
|
||||
if r.WindowMinutes < 1 {
|
||||
return errors.New("window must be at least 1 minute")
|
||||
}
|
||||
if r.Percent < 1 || r.Percent > 100 {
|
||||
return errors.New("percent must be between 1 and 100")
|
||||
}
|
||||
if r.CoresX10 < 1 || r.CoresX10 > 10 {
|
||||
return errors.New("cores must be between 0.1 and 1")
|
||||
}
|
||||
if r.Enabled && r.DurationSeconds <= 0 {
|
||||
return errors.New("duration must be greater than 0 when the rule is enabled")
|
||||
}
|
||||
if r.DurationSeconds < 0 {
|
||||
return errors.New("duration must not be negative")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CPULimitRule returns the stored rule, or the disabled default when unset.
|
||||
func (m *Manager) CPULimitRule() (CPULimitRule, error) {
|
||||
return m.loadCPULimitRule()
|
||||
}
|
||||
|
||||
// SetCPULimitRule validates and persists the rule.
|
||||
func (m *Manager) SetCPULimitRule(r CPULimitRule) error {
|
||||
if err := ValidateCPULimitRule(r); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.db.SetSetting(db.SettingCPULimitRule, string(b))
|
||||
}
|
||||
|
||||
// CPULimits returns the containers currently carrying a dynamic CPU limit,
|
||||
// keyed by container name. Safe to call from panel goroutines.
|
||||
func (m *Manager) CPULimits() map[string]CPULimitState {
|
||||
active, err := m.loadCPULimits()
|
||||
if err != nil {
|
||||
return map[string]CPULimitState{}
|
||||
}
|
||||
return active
|
||||
}
|
||||
|
||||
func (m *Manager) loadCPULimitRule() (CPULimitRule, error) {
|
||||
v, ok, err := m.db.GetSetting(db.SettingCPULimitRule)
|
||||
if err != nil {
|
||||
return CPULimitRule{}, err
|
||||
}
|
||||
if !ok || v == "" {
|
||||
return DefaultCPULimitRule(), nil
|
||||
}
|
||||
var r CPULimitRule
|
||||
if err := json.Unmarshal([]byte(v), &r); err != nil {
|
||||
return CPULimitRule{}, fmt.Errorf("parse cpu limit rule: %w", err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (m *Manager) loadCPULimits() (map[string]CPULimitState, error) {
|
||||
v, ok, err := m.db.GetSetting(db.SettingCPULimitActive)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok || v == "" {
|
||||
return map[string]CPULimitState{}, nil
|
||||
}
|
||||
var active map[string]CPULimitState
|
||||
if err := json.Unmarshal([]byte(v), &active); err != nil {
|
||||
return nil, fmt.Errorf("parse active cpu limits: %w", err)
|
||||
}
|
||||
if active == nil {
|
||||
active = map[string]CPULimitState{}
|
||||
}
|
||||
return active, nil
|
||||
}
|
||||
|
||||
func (m *Manager) saveCPULimits(active map[string]CPULimitState) error {
|
||||
if active == nil {
|
||||
active = map[string]CPULimitState{}
|
||||
}
|
||||
b, err := json.Marshal(active)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.db.SetSetting(db.SettingCPULimitActive, string(b))
|
||||
}
|
||||
|
||||
// EnforceCPULimits applies and expires the dynamic CPU limit for every
|
||||
// container. It restores the normal quota of containers whose limit expired,
|
||||
// whose user was deleted, or while the rule is disabled, then caps any
|
||||
// container that has been over Percent of its quota for WindowMinutes straight.
|
||||
// Called by the 60s sampler (single goroutine) and by the admin handler on a
|
||||
// rule change; cpuLimitMu serializes the two. Restoring uses the user's current
|
||||
// DB quota (users.cpu), so an admin quota edit during a limit wins.
|
||||
func (m *Manager) EnforceCPULimits() error {
|
||||
m.cpuLimitMu.Lock()
|
||||
defer m.cpuLimitMu.Unlock()
|
||||
|
||||
rule, err := m.loadCPULimitRule()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
active, err := m.loadCPULimits()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := m.db.ListUsers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byName := make(map[string]*db.User, len(users))
|
||||
for _, u := range users {
|
||||
byName[u.Name] = u
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
step := int64(ResourceSampleInterval / time.Second)
|
||||
var firstErr error
|
||||
changed := false
|
||||
|
||||
// Restore/forget limits that expired, belong to a deleted user, or are
|
||||
// orphaned by the rule being switched off.
|
||||
for name, st := range active {
|
||||
u := byName[name]
|
||||
if u == nil {
|
||||
delete(active, name)
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
if !rule.Enabled || now >= st.Until {
|
||||
if err := m.lx.SetCPU(name, u.CPU); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("restore cpu quota for %s: %w", name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
delete(active, name)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
|
||||
// Apply the limit to containers that stayed over the threshold.
|
||||
if rule.Enabled {
|
||||
since := now - int64(rule.WindowMinutes+2)*step
|
||||
samples, err := m.db.RecentResourceSamples(since)
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
} else {
|
||||
grouped := make(map[int64][]db.ResourceSample)
|
||||
for _, s := range samples {
|
||||
grouped[s.UserID] = append(grouped[s.UserID], s)
|
||||
}
|
||||
threshold := int64(rule.Percent) * 10
|
||||
for _, u := range users {
|
||||
if _, ok := active[u.Name]; ok {
|
||||
continue
|
||||
}
|
||||
// A limit at or above the container's own quota would raise it
|
||||
// instead of restricting it, so skip those.
|
||||
if u.CPU <= rule.CoresX10 {
|
||||
continue
|
||||
}
|
||||
if !cpuOverStreak(grouped[u.ID], now, rule.WindowMinutes, threshold) {
|
||||
continue
|
||||
}
|
||||
if err := m.lx.SetCPU(u.Name, rule.CoresX10); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("apply cpu limit to %s: %w", u.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
active[u.Name] = CPULimitState{Until: now + int64(rule.DurationSeconds), CoresX10: rule.CoresX10}
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if changed {
|
||||
if err := m.saveCPULimits(active); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// cpuOverStreak reports whether the user's newest `need` samples form a
|
||||
// contiguous run of running minutes, each above thresholdX10 (tenths of a
|
||||
// percent of the container's own quota). The newest sample must be recent, so
|
||||
// a sampling gap (panel down) never counts as sustained load.
|
||||
func cpuOverStreak(samples []db.ResourceSample, now int64, need int, thresholdX10 int64) bool {
|
||||
if need < 1 || len(samples) < need {
|
||||
return false
|
||||
}
|
||||
last := samples[len(samples)-1]
|
||||
if now-last.SampleMinute > 120 {
|
||||
return false
|
||||
}
|
||||
step := int64(ResourceSampleInterval / time.Second)
|
||||
expected := last.SampleMinute
|
||||
count := 0
|
||||
for i := len(samples) - 1; i >= 0; i-- {
|
||||
s := samples[i]
|
||||
if s.SampleMinute != expected {
|
||||
break
|
||||
}
|
||||
if s.State != resourceRunning || s.CPUPercentX10 <= thresholdX10 {
|
||||
break
|
||||
}
|
||||
count++
|
||||
if count >= need {
|
||||
return true
|
||||
}
|
||||
expected -= step
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"vpsmgr/internal/cfg"
|
||||
"vpsmgr/internal/db"
|
||||
)
|
||||
|
||||
func TestParseLimitCores(t *testing.T) {
|
||||
valid := map[string]int{"0.1": 1, "0.5": 5, "0.9": 9, "1": 10, "1.0": 10, ".5": 5}
|
||||
for in, want := range valid {
|
||||
got, err := ParseLimitCores(in)
|
||||
if err != nil {
|
||||
t.Errorf("ParseLimitCores(%q) error: %v", in, err)
|
||||
continue
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("ParseLimitCores(%q) = %d, want %d", in, got, want)
|
||||
}
|
||||
}
|
||||
invalid := []string{"", " ", "0", "0.0", "0.05", "1.1", "1.5", "2", "-0.5", "abc", "1e2"}
|
||||
for _, in := range invalid {
|
||||
if _, err := ParseLimitCores(in); err == nil {
|
||||
t.Errorf("ParseLimitCores(%q) accepted, want error", in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCPULimitRule(t *testing.T) {
|
||||
valid := []CPULimitRule{
|
||||
{Enabled: true, WindowMinutes: 1, Percent: 100, CoresX10: 1, DurationSeconds: 60},
|
||||
{Enabled: true, WindowMinutes: 10, Percent: 60, CoresX10: 10, DurationSeconds: 9000},
|
||||
{Enabled: false, WindowMinutes: 10, Percent: 60, CoresX10: 5, DurationSeconds: 0},
|
||||
}
|
||||
for _, r := range valid {
|
||||
if err := ValidateCPULimitRule(r); err != nil {
|
||||
t.Errorf("ValidateCPULimitRule(%+v) error: %v", r, err)
|
||||
}
|
||||
}
|
||||
invalid := []CPULimitRule{
|
||||
{Enabled: true, WindowMinutes: 0, Percent: 60, CoresX10: 5, DurationSeconds: 60},
|
||||
{Enabled: true, WindowMinutes: 10, Percent: 0, CoresX10: 5, DurationSeconds: 60},
|
||||
{Enabled: true, WindowMinutes: 10, Percent: 101, CoresX10: 5, DurationSeconds: 60},
|
||||
{Enabled: true, WindowMinutes: 10, Percent: 60, CoresX10: 0, DurationSeconds: 60},
|
||||
{Enabled: true, WindowMinutes: 10, Percent: 60, CoresX10: 11, DurationSeconds: 60},
|
||||
{Enabled: true, WindowMinutes: 10, Percent: 60, CoresX10: 5, DurationSeconds: 0},
|
||||
{Enabled: false, WindowMinutes: 10, Percent: 60, CoresX10: 5, DurationSeconds: -1},
|
||||
}
|
||||
for _, r := range invalid {
|
||||
if err := ValidateCPULimitRule(r); err == nil {
|
||||
t.Errorf("ValidateCPULimitRule(%+v) accepted, want error", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPULimitRuleRoundTrip(t *testing.T) {
|
||||
d, err := db.Open(filepath.Join(t.TempDir(), "rule.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
m := New(cfg.Default(), d)
|
||||
|
||||
// Unset: the disabled prefilled default is returned.
|
||||
got, err := m.CPULimitRule()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != DefaultCPULimitRule() {
|
||||
t.Fatalf("unset rule = %+v, want default %+v", got, DefaultCPULimitRule())
|
||||
}
|
||||
want := CPULimitRule{Enabled: true, WindowMinutes: 3, Percent: 25, CoresX10: 2, DurationSeconds: 3600}
|
||||
if err := m.SetCPULimitRule(want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err = m.CPULimitRule()
|
||||
if err != nil || got != want {
|
||||
t.Fatalf("round trip = %+v, err=%v, want %+v", got, err, want)
|
||||
}
|
||||
if err := m.SetCPULimitRule(CPULimitRule{Enabled: true, WindowMinutes: 0}); err == nil {
|
||||
t.Fatal("invalid rule accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUOverStreak(t *testing.T) {
|
||||
const step = 60
|
||||
now := int64(6000)
|
||||
running := func(min int64, pct int64) db.ResourceSample {
|
||||
return db.ResourceSample{SampleMinute: min, State: 1, CPUPercentX10: pct}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
samples []db.ResourceSample
|
||||
need int
|
||||
thr int64
|
||||
want bool
|
||||
}{
|
||||
{"three contiguous over", []db.ResourceSample{running(now-180, 700), running(now-120, 800), running(now-60, 900)}, 3, 600, true},
|
||||
{"newest not fresh", []db.ResourceSample{running(now-600, 700), running(now-540, 800), running(now-480, 900)}, 3, 600, false},
|
||||
{"gap breaks run", []db.ResourceSample{running(now-180, 700), running(now-120, 800), running(now, 900)}, 3, 600, false},
|
||||
{"equal threshold not over", []db.ResourceSample{running(now-120, 600), running(now-60, 600)}, 2, 600, false},
|
||||
{"stopped breaks run", []db.ResourceSample{running(now-120, 700), {SampleMinute: now - 60, State: 0, CPUPercentX10: 700}}, 2, 600, false},
|
||||
{"need more than present", []db.ResourceSample{running(now-60, 700)}, 3, 600, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := cpuOverStreak(tc.samples, now, tc.need, tc.thr); got != tc.want {
|
||||
t.Errorf("%s: cpuOverStreak = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceCPULimitsDisabledNoActive(t *testing.T) {
|
||||
d, err := db.Open(filepath.Join(t.TempDir(), "off.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
if _, err := d.CreateUser("alice", "h", "10.115.0.2", 1, 30001, 10000, 40, 1024, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := cfg.Default()
|
||||
c.Incus.Socket = "/nonexistent/vpsmgr-test.sock"
|
||||
m := New(c, d)
|
||||
|
||||
// Rule unset (disabled), nothing active: no Incus call, no error.
|
||||
if err := m.EnforceCPULimits(); err != nil {
|
||||
t.Fatalf("EnforceCPULimits with rule off: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceCPULimitsCleansDeletedUser(t *testing.T) {
|
||||
d, err := db.Open(filepath.Join(t.TempDir(), "del.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
c := cfg.Default()
|
||||
c.Incus.Socket = "/nonexistent/vpsmgr-test.sock"
|
||||
m := New(c, d)
|
||||
|
||||
active := map[string]CPULimitState{"ghost": {Until: 1 << 40, CoresX10: 5}}
|
||||
b, _ := json.Marshal(active)
|
||||
if err := d.SetSetting(db.SettingCPULimitActive, string(b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The user does not exist, so the entry is dropped without touching Incus.
|
||||
if err := m.EnforceCPULimits(); err != nil {
|
||||
t.Fatalf("EnforceCPULimits: %v", err)
|
||||
}
|
||||
if len(m.CPULimits()) != 0 {
|
||||
t.Fatalf("active limits = %+v, want empty", m.CPULimits())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceCPULimitsRestoreFailurePropagates(t *testing.T) {
|
||||
d, err := db.Open(filepath.Join(t.TempDir(), "restore.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
u, err := d.CreateUser("bob", "h", "10.115.0.3", 2, 30002, 10100, 40, 1024, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := cfg.Default()
|
||||
c.Incus.Socket = "/nonexistent/vpsmgr-test.sock"
|
||||
m := New(c, d)
|
||||
|
||||
active := map[string]CPULimitState{u.Name: {Until: 1, CoresX10: 5}} // already expired
|
||||
b, _ := json.Marshal(active)
|
||||
if err := d.SetSetting(db.SettingCPULimitActive, string(b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.EnforceCPULimits(); err == nil {
|
||||
t.Fatal("expected an error: restoring an expired limit must reach Incus")
|
||||
}
|
||||
// The restore failed, so the entry is kept for the next pass.
|
||||
if _, ok := m.CPULimits()[u.Name]; !ok {
|
||||
t.Fatal("active entry dropped despite a failed restore")
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,13 @@ type Manager struct {
|
||||
// deliberately NOT held while Del removes domain files — Del already holds
|
||||
// opMu, and holding both would risk lock-order deadlock with AddDomain.
|
||||
domainMu sync.Mutex
|
||||
|
||||
// cpuLimitMu serializes the dynamic CPU limit enforcement. It is held by
|
||||
// the 60s sampler and by the admin handler when a rule change is applied
|
||||
// immediately, so a rule toggle cannot race with the periodic pass. The
|
||||
// active-limit state itself lives in the DB (SettingCPULimitActive), so a
|
||||
// panel restart keeps every countdown and still restores on expiry.
|
||||
cpuLimitMu sync.Mutex
|
||||
}
|
||||
|
||||
func New(c *cfg.Config, d *db.DB) *Manager {
|
||||
|
||||
@@ -157,6 +157,8 @@ type pageData struct {
|
||||
BandwidthPct int // used/quota * 100, clamped to 100
|
||||
BandwidthResetDay int // day of month the bandwidth period resets (1-28)
|
||||
Throttled bool // over quota: NIC limited to 1Mbps
|
||||
CPULimited bool // under the admin's dynamic CPU limit
|
||||
CPULimitUntil int64 // unix seconds the dynamic CPU limit expires
|
||||
Domains []domainRow
|
||||
QuotaCPU string
|
||||
QuotaMem string
|
||||
@@ -372,6 +374,13 @@ func (s *Server) buildData(u *db.User, msg, errMsg string) pageData {
|
||||
// Resource usage comes from the persisted sampler snapshot (five-minute
|
||||
// CPU average plus latest memory/disk), never from a live Incus sample.
|
||||
d.CPUUse, d.MemUse, d.DiskUsed = s.mgr.PanelResources(u.ID)
|
||||
// Dynamic CPU limit (admin global rule): while capped, show the cap instead
|
||||
// of the quota and hand the template an expiry for its live countdown.
|
||||
if st, ok := s.mgr.CPULimits()[u.Name]; ok && st.Until > time.Now().Unix() {
|
||||
d.QuotaCPU = mgr.FormatCPU(st.CoresX10)
|
||||
d.CPULimited = true
|
||||
d.CPULimitUntil = st.Until
|
||||
}
|
||||
// One `incus list` call only for the container status (must be live).
|
||||
// Bandwidth is read from the DB — the background sampler writes it every 60s.
|
||||
st, err := s.mgr.State(u.Name)
|
||||
|
||||
@@ -416,6 +416,33 @@ func TestOverviewShowsBandwidthQuota(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOverviewShowsCPULimit verifies the dynamic CPU limit badge (capped core
|
||||
// count plus the expiry the frontend counts down) renders while a limit is
|
||||
// active and stays absent otherwise.
|
||||
func TestOverviewShowsCPULimit(t *testing.T) {
|
||||
srv, _ := newTestServer(t)
|
||||
limited := srv.renderToString(t, "overview.html", pageData{
|
||||
User: &db.User{Name: "alice"},
|
||||
Prefix: "/" + testSecret,
|
||||
QuotaCPU: "0.5",
|
||||
CPULimited: true,
|
||||
CPULimitUntil: 4102444800,
|
||||
})
|
||||
for _, want := range []string{`id="cpuLimitPill"`, `data-until="4102444800"`, "0.5"} {
|
||||
if !strings.Contains(limited, want) {
|
||||
t.Errorf("limited overview missing %q", want)
|
||||
}
|
||||
}
|
||||
plain := srv.renderToString(t, "overview.html", pageData{
|
||||
User: &db.User{Name: "alice"},
|
||||
Prefix: "/" + testSecret,
|
||||
QuotaCPU: "4",
|
||||
})
|
||||
if strings.Contains(plain, `id="cpuLimitPill"`) {
|
||||
t.Error("unlimited overview should not render the CPU limit badge")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOverviewThemeColor verifies the operator-assigned accent color is
|
||||
// injected as a CSS variable override block only when set: with a theme color
|
||||
// the overview carries the tinted background and recolored accents, without it
|
||||
|
||||
@@ -257,7 +257,7 @@ footer a{color:var(--muted);text-decoration:underline}
|
||||
<div class="card">
|
||||
<h2>{{if eq .Lang "zh"}}机器管理{{else}}Machine{{end}}{{if eq .State "Running"}}<span class="pill up" style="margin-left:.5rem">{{if eq .Lang "zh"}}运行中{{else}}Running{{end}}</span>{{else}}<span class="pill down" style="margin-left:.5rem">{{.State}}</span>{{end}}</h2>
|
||||
<div class="qgrid">
|
||||
<div class="qitem"><span class="qlabel">CPU</span><span class="qval">{{.QuotaCPU}}</span></div>
|
||||
<div class="qitem"><span class="qlabel">CPU</span><span class="qval">{{.QuotaCPU}}{{if .CPULimited}} <span class="pill down" id="cpuLimitPill" data-until="{{.CPULimitUntil}}" title="{{if eq .Lang "zh"}}受管理员 CPU 动态限制规则限制,到期自动恢复原配额{{else}}capped by the admin's dynamic CPU limit rule; the normal quota returns when the countdown ends{{end}}">{{if eq .Lang "zh"}}限制中{{else}}limited{{end}} <span id="cpuLimitCount"></span></span>{{end}}</span></div>
|
||||
<div class="qitem"><span class="qlabel">{{if eq .Lang "zh"}}内存{{else}}Memory{{end}}</span><span class="qval">{{.QuotaMem}}</span></div>
|
||||
<div class="qitem"><span class="qlabel">{{if eq .Lang "zh"}}磁盘{{else}}Disk{{end}}</span><span class="qval">{{.QuotaDisk}}</span></div>
|
||||
</div>
|
||||
@@ -550,6 +550,24 @@ apt-get update && apt-get install -y nginx{{end}}">{{.InitScript}}</textarea>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Dynamic CPU limit countdown: while capped, tick down the remaining time
|
||||
// locally from the server-provided expiry; when it reaches zero, reload so the
|
||||
// page shows the restored quota.
|
||||
(function(){
|
||||
var pill=document.getElementById('cpuLimitPill');
|
||||
if(!pill)return;
|
||||
var out=document.getElementById('cpuLimitCount');
|
||||
var until=parseInt(pill.getAttribute('data-until'),10)*1000;
|
||||
function pad(n){return (n<10?'0':'')+n;}
|
||||
function tick(){
|
||||
var left=Math.floor((until-Date.now())/1000);
|
||||
if(left<=0){ if(out)out.textContent=''; location.reload(); return; }
|
||||
var h=Math.floor(left/3600), m=Math.floor((left%3600)/60), s=left%60;
|
||||
if(out)out.textContent=h>0?(h+':'+pad(m)+':'+pad(s)):(pad(m)+':'+pad(s));
|
||||
}
|
||||
tick();
|
||||
setInterval(tick,1000);
|
||||
})();
|
||||
(function(){
|
||||
var P='{{.Prefix}}';
|
||||
function toast(msg,isErr){
|
||||
|
||||
+8
-1
@@ -734,7 +734,8 @@ func pathUnder(path, prefix string) bool {
|
||||
// the panels, so opening a page never triggers live Incus sampling. After every
|
||||
// sample it enforces bandwidth quotas: users over their monthly limit get their
|
||||
// NIC rate-limited to 1Mbps, users back under (e.g. monthly rollover) get the
|
||||
// limit removed.
|
||||
// limit removed. It then enforces the global dynamic CPU limit rule, capping
|
||||
// containers that stayed over their quota and restoring expired ones.
|
||||
func sampleResourceLoop(m *mgr.Manager) {
|
||||
if err := m.SampleResources(); err != nil {
|
||||
log.Printf("resource sample: %v", err)
|
||||
@@ -743,6 +744,9 @@ func sampleResourceLoop(m *mgr.Manager) {
|
||||
if err := m.EnforceBandwidthLimits(); err != nil {
|
||||
log.Printf("bandwidth quota enforcement: %v", err)
|
||||
}
|
||||
if err := m.EnforceCPULimits(); err != nil {
|
||||
log.Printf("cpu dynamic limit enforcement: %v", err)
|
||||
}
|
||||
tick := time.NewTicker(mgr.BandwidthInterval)
|
||||
defer tick.Stop()
|
||||
for range tick.C {
|
||||
@@ -753,6 +757,9 @@ func sampleResourceLoop(m *mgr.Manager) {
|
||||
if err := m.EnforceBandwidthLimits(); err != nil {
|
||||
log.Printf("bandwidth quota enforcement: %v", err)
|
||||
}
|
||||
if err := m.EnforceCPULimits(); err != nil {
|
||||
log.Printf("cpu dynamic limit enforcement: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user