From baa6266fdfe9482e1a86cb0fb60cc20948d236ff Mon Sep 17 00:00:00 2001 From: CN_CoreSteb <74492810+CNCoreSteb@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:06:09 +1000 Subject: [PATCH 1/2] perf(monitor): improve monitor query and chart performance for large ranges - enable LTTB sampling in the shared line chart to bound rendered points - empty io/network in monitor search now means no name filter, returning all device rows in one query instead of fanning out per device - merge device names recorded in the monitor db into io/network/gpu options so removed devices stay selectable; move logic to service layer - add composite indexes (name, created_at) on monitor_ios/monitor_networks and (product_name, created_at) on monitor_gpus - stop migrating the unused MonitorGPU model into MonitorDB - fix the memory param which was compared against "mem" and never matched --- agent/app/api/v2/monitor.go | 22 +- agent/app/repo/monitor.go | 21 ++ agent/app/service/monitor.go | 93 ++++++- agent/app/service/monitor_test.go | 244 ++++++++++++++++++ agent/init/hook/hook.go | 34 ++- .../components/v-charts/components/Line.vue | 2 + .../src/views/host/monitor/monitor/index.vue | 8 +- 7 files changed, 384 insertions(+), 40 deletions(-) create mode 100644 agent/app/service/monitor_test.go diff --git a/agent/app/api/v2/monitor.go b/agent/app/api/v2/monitor.go index e1c8ecc07d79..f88a4661cda6 100644 --- a/agent/app/api/v2/monitor.go +++ b/agent/app/api/v2/monitor.go @@ -1,13 +1,9 @@ package v2 import ( - "sort" - "github.com/1Panel-dev/1Panel/agent/app/api/v2/helper" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/gin-gonic/gin" - "github.com/shirou/gopsutil/v4/disk" - "github.com/shirou/gopsutil/v4/net" ) // @Tags Monitor @@ -91,14 +87,7 @@ func (b *BaseApi) UpdateMonitorSetting(c *gin.Context) { // @Security Timestamp // @Router /hosts/monitor/netoptions [get] func (b *BaseApi) GetNetworkOptions(c *gin.Context) { - netStat, _ := net.IOCounters(true) - var options []string - options = append(options, "all") - for _, net := range netStat { - options = append(options, net.Name) - } - sort.Strings(options) - helper.SuccessWithData(c, options) + helper.SuccessWithData(c, monitorService.LoadNetworkOptions()) } // @Tags Monitor @@ -108,12 +97,5 @@ func (b *BaseApi) GetNetworkOptions(c *gin.Context) { // @Security Timestamp // @Router /hosts/monitor/iooptions [get] func (b *BaseApi) GetIOOptions(c *gin.Context) { - diskStat, _ := disk.IOCounters() - var options []string - options = append(options, "all") - for _, net := range diskStat { - options = append(options, net.Name) - } - sort.Strings(options) - helper.SuccessWithData(c, options) + helper.SuccessWithData(c, monitorService.LoadIOOptions()) } diff --git a/agent/app/repo/monitor.go b/agent/app/repo/monitor.go index 3814252283cc..0da3643feee3 100644 --- a/agent/app/repo/monitor.go +++ b/agent/app/repo/monitor.go @@ -15,6 +15,9 @@ type IMonitorRepo interface { GetGPU(opts ...DBOption) ([]model.MonitorGPU, error) GetIO(opts ...DBOption) ([]model.MonitorIO, error) GetNetwork(opts ...DBOption) ([]model.MonitorNetwork, error) + GetIONames() ([]string, error) + GetNetworkNames() ([]string, error) + GetGPUProductNames() ([]string, error) CreateMonitorBase(model model.MonitorBase) error BatchCreateMonitorGPU(list []model.MonitorGPU) error @@ -69,6 +72,24 @@ func (u *MonitorRepo) GetGPU(opts ...DBOption) ([]model.MonitorGPU, error) { return data, err } +func (u *MonitorRepo) GetIONames() ([]string, error) { + var names []string + err := global.MonitorDB.Model(&model.MonitorIO{}).Distinct().Pluck("name", &names).Error + return names, err +} + +func (u *MonitorRepo) GetNetworkNames() ([]string, error) { + var names []string + err := global.MonitorDB.Model(&model.MonitorNetwork{}).Distinct().Pluck("name", &names).Error + return names, err +} + +func (u *MonitorRepo) GetGPUProductNames() ([]string, error) { + var names []string + err := global.GPUMonitorDB.Model(&model.MonitorGPU{}).Distinct().Pluck("product_name", &names).Error + return names, err +} + func (u *MonitorRepo) CreateMonitorBase(model model.MonitorBase) error { return global.MonitorDB.Create(&model).Error } diff --git a/agent/app/service/monitor.go b/agent/app/service/monitor.go index 1f4ffe66e6ec..1a7cc6033bf0 100644 --- a/agent/app/service/monitor.go +++ b/agent/app/service/monitor.go @@ -49,6 +49,8 @@ type IMonitorService interface { UpdateSetting(key, value string) error CleanData() error + LoadIOOptions() []string + LoadNetworkOptions() []string LoadGPUOptions() dto.MonitorGPUOptions LoadGPUMonitorData(req dto.MonitorGPUSearch) (dto.MonitorGPUData, error) @@ -85,7 +87,7 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa base.TopCPUItems = processes base.TopCPU = "" } - if req.Param == "all" || req.Param == "mem" { + if req.Param == "all" || req.Param == "memory" { var processes []dto.Process _ = json.Unmarshal([]byte(base.TopMem), &processes) base.TopMemItems = processes @@ -96,7 +98,11 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa data = append(data, itemData) } if req.Param == "all" || req.Param == "io" { - bases, err := monitorRepo.GetIO(repo.WithByName(req.IO), repo.WithByCreatedAt(req.StartTime, req.EndTime)) + ioOpts := []repo.DBOption{repo.WithByCreatedAt(req.StartTime, req.EndTime)} + if len(req.IO) != 0 { + ioOpts = append(ioOpts, repo.WithByName(req.IO)) + } + bases, err := monitorRepo.GetIO(ioOpts...) if err != nil { return nil, err } @@ -110,7 +116,11 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa data = append(data, itemData) } if req.Param == "all" || req.Param == "network" { - bases, err := monitorRepo.GetNetwork(repo.WithByName(req.Network), repo.WithByCreatedAt(req.StartTime, req.EndTime)) + netOpts := []repo.DBOption{repo.WithByCreatedAt(req.StartTime, req.EndTime)} + if len(req.Network) != 0 { + netOpts = append(netOpts, repo.WithByName(req.Network)) + } + bases, err := monitorRepo.GetNetwork(netOpts...) if err != nil { return nil, err } @@ -126,21 +136,78 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa return data, nil } +func (m *MonitorService) LoadIOOptions() []string { + optionSet := make(map[string]struct{}) + if diskStat, err := disk.IOCounters(); err == nil { + for _, item := range diskStat { + optionSet[item.Name] = struct{}{} + } + } + // union with names recorded in the monitor db so removed devices stay selectable + if names, err := monitorRepo.GetIONames(); err == nil { + for _, name := range names { + optionSet[name] = struct{}{} + } + } + return sortedMonitorOptions(optionSet) +} + +func (m *MonitorService) LoadNetworkOptions() []string { + optionSet := make(map[string]struct{}) + if netStat, err := net.IOCounters(true); err == nil { + for _, item := range netStat { + optionSet[item.Name] = struct{}{} + } + } + if names, err := monitorRepo.GetNetworkNames(); err == nil { + for _, name := range names { + optionSet[name] = struct{}{} + } + } + return sortedMonitorOptions(optionSet) +} + +func sortedMonitorOptions(optionSet map[string]struct{}) []string { + options := make([]string, 0, len(optionSet)) + for name := range optionSet { + if len(name) != 0 && name != "all" { + options = append(options, name) + } + } + sort.Strings(options) + return append([]string{"all"}, options...) +} + func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions { var data dto.MonitorGPUOptions exist, client := accelerator.New() - if !exist { - return data - } - snapshot, err := client.Collect(context.Background()) - if err != nil { - global.LOG.Errorf("Load accelerator info failed, err: %v", err) - return data + if exist { + snapshot, err := client.Collect(context.Background()) + if err != nil { + global.LOG.Errorf("Load accelerator info failed, err: %v", err) + } else { + if warning := snapshot.Warning(); warning != nil { + global.LOG.Warnf("Load accelerator info partially failed, err: %v", warning) + } + data = loadGPUOptions(snapshot) + } } - if warning := snapshot.Warning(); warning != nil { - global.LOG.Warnf("Load accelerator info partially failed, err: %v", warning) + // union with product names recorded in the monitor db so removed accelerators stay selectable + if names, err := monitorRepo.GetGPUProductNames(); err == nil { + present := make(map[string]struct{}, len(data.Options)) + for _, name := range data.Options { + present[name] = struct{}{} + } + var extra []string + for _, name := range names { + if _, ok := present[name]; !ok && len(name) != 0 { + extra = append(extra, name) + } + } + sort.Strings(extra) + data.Options = append(data.Options, extra...) } - return loadGPUOptions(snapshot) + return data } func loadGPUOptions(snapshot *accelerator.Snapshot) dto.MonitorGPUOptions { diff --git a/agent/app/service/monitor_test.go b/agent/app/service/monitor_test.go new file mode 100644 index 000000000000..5c918f749580 --- /dev/null +++ b/agent/app/service/monitor_test.go @@ -0,0 +1,244 @@ +package service + +import ( + "io" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/re" + "github.com/glebarez/sqlite" + "github.com/sirupsen/logrus" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" +) + +func setupMonitorDBs(t *testing.T) { + t.Helper() + dir := t.TempDir() + cfg := &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)} + + monitorDB, err := gorm.Open(sqlite.Open(filepath.Join(dir, "monitor.db")), cfg) + if err != nil { + t.Fatalf("open monitor db: %v", err) + } + gpuDB, err := gorm.Open(sqlite.Open(filepath.Join(dir, "gpu_monitor.db")), cfg) + if err != nil { + t.Fatalf("open gpu monitor db: %v", err) + } + if err := monitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorNetwork{}, &model.MonitorIO{}); err != nil { + t.Fatalf("migrate monitor db: %v", err) + } + if err := gpuDB.AutoMigrate(&model.MonitorGPU{}); err != nil { + t.Fatalf("migrate gpu monitor db: %v", err) + } + for _, stmt := range []string{ + "CREATE INDEX IF NOT EXISTS idx_monitor_ios_name_created ON monitor_ios(name, created_at)", + "CREATE INDEX IF NOT EXISTS idx_monitor_networks_name_created ON monitor_networks(name, created_at)", + } { + if err := monitorDB.Exec(stmt).Error; err != nil { + t.Fatalf("create index: %v", err) + } + } + if err := gpuDB.Exec("CREATE INDEX IF NOT EXISTS idx_monitor_gpus_product_created ON monitor_gpus(product_name, created_at)").Error; err != nil { + t.Fatalf("create gpu index: %v", err) + } + global.MonitorDB = monitorDB + global.GPUMonitorDB = gpuDB + + // production binaries run these during server init; tests must do it explicitly + re.Init() + if global.LOG == nil { + global.LOG = logrus.New() + global.LOG.SetOutput(io.Discard) + } +} + +func seedMonitorRows(t *testing.T) time.Time { + t.Helper() + // sqlite stores timestamps as RFC3339 text and the service converts query bounds to the + // local zone, so seeds must use local time for range comparisons to match + base := time.Now().Add(-time.Minute).Truncate(time.Second) + ioRows := []model.MonitorIO{ + {BaseModel: model.BaseModel{CreatedAt: base}, Name: "all", Read: 1, Write: 1}, + {BaseModel: model.BaseModel{CreatedAt: base}, Name: "sda", Read: 2, Write: 3}, + {BaseModel: model.BaseModel{CreatedAt: base}, Name: "veth-old", Read: 4, Write: 5}, + {BaseModel: model.BaseModel{CreatedAt: base.Add(time.Minute)}, Name: "sda", Read: 6, Write: 7}, + } + if err := global.MonitorDB.CreateInBatches(ioRows, len(ioRows)).Error; err != nil { + t.Fatalf("seed io: %v", err) + } + netRows := []model.MonitorNetwork{ + {BaseModel: model.BaseModel{CreatedAt: base}, Name: "all", Up: 1, Down: 1}, + {BaseModel: model.BaseModel{CreatedAt: base}, Name: "eth0", Up: 2, Down: 3}, + {BaseModel: model.BaseModel{CreatedAt: base}, Name: "veth-old", Up: 4, Down: 5}, + } + if err := global.MonitorDB.CreateInBatches(netRows, len(netRows)).Error; err != nil { + t.Fatalf("seed network: %v", err) + } + gpuRows := []model.MonitorGPU{ + {BaseModel: model.BaseModel{CreatedAt: base}, ProductName: "NVIDIA RTX 4090", GPUUtil: 10}, + {BaseModel: model.BaseModel{CreatedAt: base}, ProductName: "NVIDIA RTX 3090", GPUUtil: 20}, + } + if err := global.GPUMonitorDB.CreateInBatches(gpuRows, len(gpuRows)).Error; err != nil { + t.Fatalf("seed gpu: %v", err) + } + baseRows := []model.MonitorBase{ + {BaseModel: model.BaseModel{CreatedAt: base}, Cpu: 10, Memory: 20, LoadUsage: 30}, + } + if err := global.MonitorDB.CreateInBatches(baseRows, len(baseRows)).Error; err != nil { + t.Fatalf("seed base: %v", err) + } + return base +} + +func searchRange(base time.Time) (time.Time, time.Time) { + return base.Add(-time.Hour), base.Add(time.Hour) +} + +func ioRowNames(t *testing.T, items []dto.MonitorData) map[string]int { + t.Helper() + counts := make(map[string]int) + for _, item := range items { + if item.Param != "io" { + continue + } + for _, v := range item.Value { + row := v.(model.MonitorIO) + counts[row.Name]++ + } + } + return counts +} + +func TestLoadMonitorDataIOWildcard(t *testing.T) { + setupMonitorDBs(t) + base := seedMonitorRows(t) + start, end := searchRange(base) + svc := NewIMonitorService() + + all, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "io", IO: "", StartTime: start, EndTime: end}) + if err != nil { + t.Fatalf("wildcard query: %v", err) + } + counts := ioRowNames(t, all) + if counts["sda"] != 2 || counts["veth-old"] != 1 || counts["all"] != 1 { + t.Fatalf("expected all stored devices in wildcard result, got %v", counts) + } + + one, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "io", IO: "sda", StartTime: start, EndTime: end}) + if err != nil { + t.Fatalf("named query: %v", err) + } + if counts := ioRowNames(t, one); len(counts) != 1 || counts["sda"] != 2 { + t.Fatalf("expected only sda rows, got %v", counts) + } + + agg, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "io", IO: "all", StartTime: start, EndTime: end}) + if err != nil { + t.Fatalf("aggregate query: %v", err) + } + if counts := ioRowNames(t, agg); len(counts) != 1 || counts["all"] != 1 { + t.Fatalf("expected only aggregate rows, got %v", counts) + } +} + +func TestLoadMonitorDataNetworkWildcard(t *testing.T) { + setupMonitorDBs(t) + base := seedMonitorRows(t) + start, end := searchRange(base) + svc := NewIMonitorService() + + items, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "network", Network: "", StartTime: start, EndTime: end}) + if err != nil { + t.Fatalf("wildcard query: %v", err) + } + counts := make(map[string]int) + for _, item := range items { + if item.Param != "network" { + continue + } + for _, v := range item.Value { + counts[v.(model.MonitorNetwork).Name]++ + } + } + if counts["eth0"] != 1 || counts["veth-old"] != 1 || counts["all"] != 1 { + t.Fatalf("expected all stored interfaces, got %v", counts) + } +} + +func TestLoadMonitorDataMemoryParam(t *testing.T) { + setupMonitorDBs(t) + base := seedMonitorRows(t) + start, end := searchRange(base) + svc := NewIMonitorService() + + items, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "memory", IO: "all", Network: "all", StartTime: start, EndTime: end}) + if err != nil { + t.Fatalf("memory query: %v", err) + } + var baseItem *dto.MonitorData + for i := range items { + if items[i].Param == "base" { + baseItem = &items[i] + } + } + if baseItem == nil || len(baseItem.Value) != 1 { + t.Fatalf("expected one base row for param=memory, got %+v", items) + } +} + +func TestMonitorOptionsIncludeRemovedDevices(t *testing.T) { + setupMonitorDBs(t) + seedMonitorRows(t) + svc := NewIMonitorService() + + contains := func(list []string, want string) bool { + for _, item := range list { + if item == want { + return true + } + } + return false + } + ioOpts := svc.LoadIOOptions() + if ioOpts[0] != "all" || !contains(ioOpts, "veth-old") || !contains(ioOpts, "sda") { + t.Fatalf("io options should keep historical devices with 'all' first, got %v", ioOpts) + } + if !sort.StringsAreSorted(ioOpts[1:]) { + t.Fatalf("io options after 'all' should be sorted, got %v", ioOpts) + } + netOpts := svc.LoadNetworkOptions() + if netOpts[0] != "all" || !contains(netOpts, "veth-old") { + t.Fatalf("network options should keep historical interfaces, got %v", netOpts) + } + gpuOpts := svc.LoadGPUOptions() + if !contains(gpuOpts.Options, "NVIDIA RTX 4090") || !contains(gpuOpts.Options, "NVIDIA RTX 3090") { + t.Fatalf("gpu options should keep historical products, got %v", gpuOpts.Options) + } +} + +func TestMonitorCompositeIndexesExist(t *testing.T) { + setupMonitorDBs(t) + seedMonitorRows(t) + for _, name := range []string{"idx_monitor_ios_name_created", "idx_monitor_networks_name_created"} { + var count int64 + if err := global.MonitorDB.Raw("SELECT COUNT(1) FROM sqlite_master WHERE type='index' AND name=?", name).Scan(&count).Error; err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + if count != 1 { + t.Fatalf("index %s missing", name) + } + } + var count int64 + if err := global.GPUMonitorDB.Raw("SELECT COUNT(1) FROM sqlite_master WHERE type='index' AND name='idx_monitor_gpus_product_created'").Scan(&count).Error; err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + if count != 1 { + t.Fatal("index idx_monitor_gpus_product_created missing") + } +} diff --git a/agent/init/hook/hook.go b/agent/init/hook/hook.go index 57fb7fe83fed..f9a907e75ac5 100644 --- a/agent/init/hook/hook.go +++ b/agent/init/hook/hook.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "strings" + "time" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" @@ -13,6 +14,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/alert_push" "github.com/1Panel-dev/1Panel/agent/utils/xpack" + "gorm.io/gorm" ) func Init() { @@ -174,6 +176,36 @@ func initAlertTask() { } func initMonitorDB() { - _ = global.MonitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorNetwork{}, &model.MonitorGPU{}, &model.MonitorIO{}) + _ = global.MonitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorNetwork{}, &model.MonitorIO{}) + _ = global.GPUMonitorDB.AutoMigrate(&model.MonitorGPU{}) _ = global.TaskDB.AutoMigrate(&model.Task{}) + // building indexes on large monitor tables can take seconds, keep it off the startup path; + // WAL mode leaves readers unblocked and busy_timeout covers the collector's inserts meanwhile + go ensureMonitorIndexes() +} + +func ensureMonitorIndexes() { + indexes := []struct { + db *gorm.DB + stmt string + }{ + // created_at alone serves unfiltered range queries and retention cleanup + {global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_bases_created ON monitor_bases(created_at)"}, + {global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_ios_created ON monitor_ios(created_at)"}, + {global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_networks_created ON monitor_networks(created_at)"}, + {global.GPUMonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_gpus_created ON monitor_gpus(created_at)"}, + // (name, created_at) serves per-device range queries and distinct name lookups + {global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_ios_name_created ON monitor_ios(name, created_at)"}, + {global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_networks_name_created ON monitor_networks(name, created_at)"}, + {global.GPUMonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_gpus_product_created ON monitor_gpus(product_name, created_at)"}, + } + start := time.Now() + for _, index := range indexes { + if err := index.db.Exec(index.stmt).Error; err != nil { + global.LOG.Warnf("create monitor index failed, stmt: %s, err: %v", index.stmt, err) + } + } + if elapsed := time.Since(start); elapsed > time.Second { + global.LOG.Infof("monitor indexes ready, took %s", elapsed) + } } diff --git a/frontend/src/components/v-charts/components/Line.vue b/frontend/src/components/v-charts/components/Line.vue index ab11b60440ef..c8fe00c36c0d 100644 --- a/frontend/src/components/v-charts/components/Line.vue +++ b/frontend/src/components/v-charts/components/Line.vue @@ -138,6 +138,7 @@ function initChart() { areaStyle: seriesStyle[index], data: item?.data, showSymbol: false, + sampling: 'lttb', yAxisIndex: item.yAxisIndex ? 1 : null, }); }); @@ -226,6 +227,7 @@ function initChart() { }; const tooltip = props.option.tooltip || defaultTooltip; const option = { + animation: (props.option.xData?.length ?? 0) <= 3000, title: [ { left: 'center', diff --git a/frontend/src/views/host/monitor/monitor/index.vue b/frontend/src/views/host/monitor/monitor/index.vue index 73a7997276bb..a86aff875e88 100644 --- a/frontend/src/views/host/monitor/monitor/index.vue +++ b/frontend/src/views/host/monitor/monitor/index.vue @@ -327,11 +327,7 @@ const searchGlobal = () => { timeRangeMemory.value = timeRangeGlobal.value; timeRangeIO.value = timeRangeGlobal.value; timeRangeNetwork.value = timeRangeGlobal.value; - search('load'); - search('cpu'); - search('memory'); - search('io'); - search('network'); + search('all'); }; const search = async (param: string) => { @@ -355,7 +351,7 @@ const search = async (param: string) => { searchInfo.network = networkChoose.value || 'all'; break; case 'all': - searchTime.value = timeRangeNetwork.value; + searchTime.value = timeRangeGlobal.value; searchInfo.io = ioChoose.value || 'all'; searchInfo.network = networkChoose.value || 'all'; break; From c298f7b88b19ce97d59a2ed7dd825731c84fd479 Mon Sep 17 00:00:00 2001 From: ssongliu Date: Tue, 29 Sep 2026 15:31:08 +0800 Subject: [PATCH 2/2] chore(monitor): remove monitor test fixtures --- agent/app/service/monitor_test.go | 262 ------------------------------ 1 file changed, 262 deletions(-) delete mode 100644 agent/app/service/monitor_test.go diff --git a/agent/app/service/monitor_test.go b/agent/app/service/monitor_test.go deleted file mode 100644 index 9b1fa17886ab..000000000000 --- a/agent/app/service/monitor_test.go +++ /dev/null @@ -1,262 +0,0 @@ -package service - -import ( - "io" - "path/filepath" - "sort" - "testing" - "time" - - "github.com/1Panel-dev/1Panel/agent/app/dto" - "github.com/1Panel-dev/1Panel/agent/app/model" - "github.com/1Panel-dev/1Panel/agent/global" - "github.com/1Panel-dev/1Panel/agent/utils/re" - "github.com/glebarez/sqlite" - "github.com/sirupsen/logrus" - "gorm.io/gorm" - gormlogger "gorm.io/gorm/logger" -) - -func setupMonitorDBs(t *testing.T) { - t.Helper() - dir := t.TempDir() - cfg := &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)} - - monitorDB, err := gorm.Open(sqlite.Open(filepath.Join(dir, "monitor.db")), cfg) - if err != nil { - t.Fatalf("open monitor db: %v", err) - } - gpuDB, err := gorm.Open(sqlite.Open(filepath.Join(dir, "gpu_monitor.db")), cfg) - if err != nil { - t.Fatalf("open gpu monitor db: %v", err) - } - if err := monitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorNetwork{}, &model.MonitorIO{}); err != nil { - t.Fatalf("migrate monitor db: %v", err) - } - if err := gpuDB.AutoMigrate(&model.MonitorGPU{}); err != nil { - t.Fatalf("migrate gpu monitor db: %v", err) - } - for _, stmt := range []string{ - "CREATE INDEX IF NOT EXISTS idx_monitor_ios_name_created ON monitor_ios(name, created_at)", - "CREATE INDEX IF NOT EXISTS idx_monitor_networks_name_created ON monitor_networks(name, created_at)", - } { - if err := monitorDB.Exec(stmt).Error; err != nil { - t.Fatalf("create index: %v", err) - } - } - if err := gpuDB.Exec("CREATE INDEX IF NOT EXISTS idx_monitor_gpus_product_created ON monitor_gpus(product_name, created_at)").Error; err != nil { - t.Fatalf("create gpu index: %v", err) - } - global.MonitorDB = monitorDB - global.GPUMonitorDB = gpuDB - - // production binaries run these during server init; tests must do it explicitly - re.Init() - if global.LOG == nil { - global.LOG = logrus.New() - global.LOG.SetOutput(io.Discard) - } -} - -func seedMonitorRows(t *testing.T) time.Time { - t.Helper() - // sqlite stores timestamps as RFC3339 text and the service converts query bounds to the - // local zone, so seeds must use local time for range comparisons to match - base := time.Now().Add(-time.Minute).Truncate(time.Second) - ioRows := []model.MonitorIO{ - {BaseModel: model.BaseModel{CreatedAt: base}, Name: "all", Read: 1, Write: 1}, - {BaseModel: model.BaseModel{CreatedAt: base}, Name: "sda", Read: 2, Write: 3}, - {BaseModel: model.BaseModel{CreatedAt: base}, Name: "veth-old", Read: 4, Write: 5}, - {BaseModel: model.BaseModel{CreatedAt: base.Add(time.Minute)}, Name: "sda", Read: 6, Write: 7}, - } - if err := global.MonitorDB.CreateInBatches(ioRows, len(ioRows)).Error; err != nil { - t.Fatalf("seed io: %v", err) - } - netRows := []model.MonitorNetwork{ - {BaseModel: model.BaseModel{CreatedAt: base}, Name: "all", Up: 1, Down: 1}, - {BaseModel: model.BaseModel{CreatedAt: base}, Name: "eth0", Up: 2, Down: 3}, - {BaseModel: model.BaseModel{CreatedAt: base}, Name: "veth-old", Up: 4, Down: 5}, - } - if err := global.MonitorDB.CreateInBatches(netRows, len(netRows)).Error; err != nil { - t.Fatalf("seed network: %v", err) - } - gpuUtil := []float64{10, 20} - gpuRows := []model.MonitorGPU{ - {BaseModel: model.BaseModel{CreatedAt: base}, ProductName: "NVIDIA RTX 4090", GPUUtil: &gpuUtil[0]}, - {BaseModel: model.BaseModel{CreatedAt: base}, ProductName: "NVIDIA RTX 3090", GPUUtil: &gpuUtil[1]}, - } - if err := global.GPUMonitorDB.CreateInBatches(gpuRows, len(gpuRows)).Error; err != nil { - t.Fatalf("seed gpu: %v", err) - } - baseRows := []model.MonitorBase{ - {BaseModel: model.BaseModel{CreatedAt: base}, Cpu: 10, Memory: 20, LoadUsage: 30}, - } - if err := global.MonitorDB.CreateInBatches(baseRows, len(baseRows)).Error; err != nil { - t.Fatalf("seed base: %v", err) - } - return base -} - -func searchRange(base time.Time) (time.Time, time.Time) { - return base.Add(-time.Hour), base.Add(time.Hour) -} - -func ioRowNames(t *testing.T, items []dto.MonitorData) map[string]int { - t.Helper() - counts := make(map[string]int) - for _, item := range items { - if item.Param != "io" { - continue - } - for _, v := range item.Value { - row := v.(model.MonitorIO) - counts[row.Name]++ - } - } - return counts -} - -func TestLoadMonitorDataIOWildcard(t *testing.T) { - setupMonitorDBs(t) - base := seedMonitorRows(t) - start, end := searchRange(base) - svc := NewIMonitorService() - - all, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "io", IO: "", StartTime: start, EndTime: end}) - if err != nil { - t.Fatalf("wildcard query: %v", err) - } - counts := ioRowNames(t, all) - if counts["sda"] != 2 || counts["veth-old"] != 1 || counts["all"] != 1 { - t.Fatalf("expected all stored devices in wildcard result, got %v", counts) - } - - one, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "io", IO: "sda", StartTime: start, EndTime: end}) - if err != nil { - t.Fatalf("named query: %v", err) - } - if counts := ioRowNames(t, one); len(counts) != 1 || counts["sda"] != 2 { - t.Fatalf("expected only sda rows, got %v", counts) - } - - agg, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "io", IO: "all", StartTime: start, EndTime: end}) - if err != nil { - t.Fatalf("aggregate query: %v", err) - } - if counts := ioRowNames(t, agg); len(counts) != 1 || counts["all"] != 1 { - t.Fatalf("expected only aggregate rows, got %v", counts) - } -} - -func TestLoadMonitorDataNetworkWildcard(t *testing.T) { - setupMonitorDBs(t) - base := seedMonitorRows(t) - start, end := searchRange(base) - svc := NewIMonitorService() - - items, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "network", Network: "", StartTime: start, EndTime: end}) - if err != nil { - t.Fatalf("wildcard query: %v", err) - } - counts := make(map[string]int) - for _, item := range items { - if item.Param != "network" { - continue - } - for _, v := range item.Value { - counts[v.(model.MonitorNetwork).Name]++ - } - } - if counts["eth0"] != 1 || counts["veth-old"] != 1 || counts["all"] != 1 { - t.Fatalf("expected all stored interfaces, got %v", counts) - } -} - -func TestLoadMonitorDataMemoryParam(t *testing.T) { - setupMonitorDBs(t) - base := seedMonitorRows(t) - start, end := searchRange(base) - svc := NewIMonitorService() - - items, err := svc.LoadMonitorData(dto.MonitorSearch{Param: "memory", IO: "all", Network: "all", StartTime: start, EndTime: end}) - if err != nil { - t.Fatalf("memory query: %v", err) - } - var baseItem *dto.MonitorData - for i := range items { - if items[i].Param == "base" { - baseItem = &items[i] - } - } - if baseItem == nil || len(baseItem.Value) != 1 { - t.Fatalf("expected one base row for param=memory, got %+v", items) - } -} - -func TestMonitorOptionsIncludeRemovedDevices(t *testing.T) { - setupMonitorDBs(t) - seedMonitorRows(t) - svc := NewIMonitorService() - - contains := func(list []string, want string) bool { - for _, item := range list { - if item == want { - return true - } - } - return false - } - ioOpts := svc.LoadIOOptions() - if ioOpts[0] != "all" || !contains(ioOpts, "veth-old") || !contains(ioOpts, "sda") { - t.Fatalf("io options should keep historical devices with 'all' first, got %v", ioOpts) - } - if !sort.StringsAreSorted(ioOpts[1:]) { - t.Fatalf("io options after 'all' should be sorted, got %v", ioOpts) - } - netOpts := svc.LoadNetworkOptions() - if netOpts[0] != "all" || !contains(netOpts, "veth-old") { - t.Fatalf("network options should keep historical interfaces, got %v", netOpts) - } - devices, err := monitorRepo.GetGPUDevices() - if err != nil { - t.Fatalf("load historical GPUs: %v", err) - } - var names []string - for _, device := range devices { - names = append(names, device.ProductName) - } - if !contains(names, "NVIDIA RTX 4090") || !contains(names, "NVIDIA RTX 3090") { - t.Fatalf("historical GPUs should remain available, got %v", names) - } - gpuOpts := svc.LoadGPUOptions() - if !gpuOpts.Supported { - if len(gpuOpts.Options) != 0 || len(gpuOpts.ChartHide) != 0 { - t.Fatalf("unsupported GPU host should return no options, got %+v", gpuOpts) - } - return - } - if !contains(gpuOpts.Options, "NVIDIA RTX 4090") || !contains(gpuOpts.Options, "NVIDIA RTX 3090") { - t.Fatalf("gpu options should keep historical products, got %v", gpuOpts.Options) - } -} - -func TestMonitorCompositeIndexesExist(t *testing.T) { - setupMonitorDBs(t) - seedMonitorRows(t) - for _, name := range []string{"idx_monitor_ios_name_created", "idx_monitor_networks_name_created"} { - var count int64 - if err := global.MonitorDB.Raw("SELECT COUNT(1) FROM sqlite_master WHERE type='index' AND name=?", name).Scan(&count).Error; err != nil { - t.Fatalf("query sqlite_master: %v", err) - } - if count != 1 { - t.Fatalf("index %s missing", name) - } - } - var count int64 - if err := global.GPUMonitorDB.Raw("SELECT COUNT(1) FROM sqlite_master WHERE type='index' AND name='idx_monitor_gpus_product_created'").Scan(&count).Error; err != nil { - t.Fatalf("query sqlite_master: %v", err) - } - if count != 1 { - t.Fatal("index idx_monitor_gpus_product_created missing") - } -}