From b3fcbc99b1347fbfba681bb5628a89e801928ea5 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:11:54 +0800 Subject: [PATCH 01/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BA=91=E4=B8=8A=E4=BC=A0=E7=AE=A1=E9=81=93=E5=A4=9A=E4=B8=AA?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E4=B8=A2=E5=A4=B1=E5=92=8C=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline 阶段修复： - 修复 Deletable 文件被后续阶段重复处理导致 MKV 覆盖 - 移除 CloudUploadStage 的 Deletable 清零循环，保留前置阶段标记 - 移除 videoUploaded 限制，支持多分段 _PART 文件全部上传 - 多视频时自动追加 .FileName 到上传路径，避免远程路径冲突 - 文件不存在时设置 uploadFailed，防止 delete_all 误删其他文件 - 模板解析失败返回空字符串而非 fallback 到错误路径 - now 模板函数改用 ctx.RecordInfo.StartTime，重试时日期稳定 Executor 修复： - 捕获 deleteMarkedFiles 返回值并写回 results，CurrentFiles 与磁盘一致 - cleanupOrphanedAssFiles 添加 .ts 扩展名，限定扫描范围为本任务文件 - StartedAt 在阶段执行前记录，耗时统计准确 OpenList 客户端修复： - 新增 APIError 结构体，isAuthError 支持 JSON code 401/403 识别 - JSON 解析失败区分 io.EOF（成功）和其他错误（失败） - Client.token 添加 sync.RWMutex 保护并发访问 - 初始密码和 token 更新改用 configs.UpdateWithRetry 原子更新 Manager 修复： - RetryTask 重置 CurrentFiles = InitialFiles，从头开始重试 - convert_mp4 添加 StderrPipe 防止 ffmpeg 挂起 前端修复： - FileProcessingPreview 使用 Form.useWatch 实时反映表单值 - OpenList 链接改用 /remotetools/tool/openlist/ 反向代理 - placeholder 使用双引号匹配 Go 模板语法 - 简洁归档预设升级为日期时间，避免同天覆盖 - 预览 Platform 使用 platformCN 与后端一致 --- src/cmd/bililive/bililive.go | 39 +++++++ src/configs/config.go | 56 +++++++--- src/pipeline/executor.go | 43 ++++++-- src/pipeline/manager.go | 1 + src/pipeline/stages/burn_subtitles.go | 7 ++ src/pipeline/stages/convert_mp4.go | 10 ++ src/pipeline/stages/custom_command.go | 7 ++ src/pipeline/stages/extract_cover.go | 100 +++++++++--------- src/pkg/openlist/client.go | 76 +++++++++---- src/pkg/openlist/manager.go | 8 +- src/recorders/recorder.go | 2 +- .../config-info/CloudUploadSettings.tsx | 35 ++++-- 12 files changed, 283 insertions(+), 101 deletions(-) diff --git a/src/cmd/bililive/bililive.go b/src/cmd/bililive/bililive.go index 23d83b8..669160c 100644 --- a/src/cmd/bililive/bililive.go +++ b/src/cmd/bililive/bililive.go @@ -366,6 +366,45 @@ func main() { logger.Info("云上传功能已启用") } + // 注册配置更新回调：当用户通过 Web UI 首次启用云上传时，自动创建并启动 OpenList Manager + configs.RegisterAfterUpdate(func(oldCfg, newCfg *configs.Config) { + if newCfg == nil { + return + } + // 检测云上传开关是否从关变为开 + wasEnabled := oldCfg != nil && oldCfg.OnRecordFinished.CloudUpload.Enable + isEnabled := newCfg.OnRecordFinished.CloudUpload.Enable + if wasEnabled || !isEnabled { + return + } + // 已有 Manager 则跳过 + if openlist.GetGlobalManager() != nil { + return + } + + logger.Info("检测到云上传已启用，正在初始化 OpenList...") + + openlistDataPath := newCfg.OpenList.DataPath + if openlistDataPath == "" { + openlistDataPath = filepath.Join(newCfg.AppDataPath, "openlist") + } + openlistPort := newCfg.OpenList.Port + if openlistPort == 0 { + openlistPort = 5244 + } + + mgr := openlist.NewManager(openlistDataPath, openlistPort) + // 先设置全局 Manager，让 API 端能立即获取到状态 + servers.SetOpenListManager(mgr) + + // 在后台启动 OpenList 进程（需要 rootCtx 生命周期） + bilisentryPkg.Go(func() { + if err := mgr.Start(rootCtx); err != nil { + logger.WithError(err).Error("动态启动 OpenList 失败，云上传功能将不可用") + } + }) + }) + // 初始化 Pipeline 管道管理器 pipelineDbPath := filepath.Join(config.AppDataPath, "db", "pipeline.db") pipelineStore, err := pipeline.NewSQLiteStore(pipelineDbPath) diff --git a/src/configs/config.go b/src/configs/config.go index 026bc58..85a2d92 100644 --- a/src/configs/config.go +++ b/src/configs/config.go @@ -618,6 +618,36 @@ var updateMu sync.Mutex // 当期望版本与实际版本不一致时返回的错误 var ErrConfigVersionConflict = errors.New("config version conflict") +// AfterUpdateCallback 配置更新后的回调函数类型 +// old 为更新前的配置（可能为 nil），newCfg 为更新后的配置 +type AfterUpdateCallback func(old, newCfg *Config) + +// afterUpdateCallbacks 配置更新后的回调列表 +var ( + afterUpdateCallbacks []AfterUpdateCallback + afterUpdateMu sync.RWMutex +) + +// RegisterAfterUpdate 注册配置更新后的回调 +// 回调在配置更新并持久化完成后同步执行，此时已释放 updateMu 锁 +func RegisterAfterUpdate(cb AfterUpdateCallback) { + afterUpdateMu.Lock() + defer afterUpdateMu.Unlock() + afterUpdateCallbacks = append(afterUpdateCallbacks, cb) +} + +// fireAfterUpdate 触发所有配置更新回调 +func fireAfterUpdate(old, newCfg *Config) { + afterUpdateMu.RLock() + callbacks := make([]AfterUpdateCallback, len(afterUpdateCallbacks)) + copy(callbacks, afterUpdateCallbacks) + afterUpdateMu.RUnlock() + + for _, cb := range callbacks { + cb(old, newCfg) + } +} + func SetCurrentConfig(cfg *Config) { if cfg == nil { // 存储 nil 以保持行为一致 @@ -657,19 +687,19 @@ func UpdateTransient(mutator func(c *Config) error) (*Config, error) { } func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) { - var newCfg *Config + var oldCfg, newCfg *Config var updateErr error func() { updateMu.Lock() defer updateMu.Unlock() - old := GetCurrentConfig() + oldCfg = GetCurrentConfig() // 若当前尚未设置配置，则以默认配置为基础 var base *Config - if old == nil { + if oldCfg == nil { base = NewConfig() } else { - base = CloneConfigShallow(old) + base = CloneConfigShallow(oldCfg) } if err := mutator(base); err != nil { updateErr = err @@ -678,10 +708,10 @@ func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) { // 维护派生字段 base.RefreshLiveRoomIndexCache() // 版本号自增 - if old == nil { + if oldCfg == nil { base.Version = 1 } else { - base.Version = old.Version + 1 + base.Version = oldCfg.Version + 1 } newCfg = base @@ -702,6 +732,7 @@ func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) { return nil, errors.New("config update failed") } + fireAfterUpdate(oldCfg, newCfg) return newCfg, nil } @@ -712,17 +743,17 @@ func UpdateCAS(expectedVersion int64, mutator func(c *Config) error) (*Config, e } func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist bool) (*Config, error) { - var newCfg *Config + var oldCfg, newCfg *Config var updateErr error func() { updateMu.Lock() defer updateMu.Unlock() - cur := GetCurrentConfig() + oldCfg = GetCurrentConfig() // 校验版本 var curVersion int64 - if cur != nil { - curVersion = cur.Version + if oldCfg != nil { + curVersion = oldCfg.Version } if curVersion != expectedVersion { updateErr = ErrConfigVersionConflict @@ -730,10 +761,10 @@ func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist } // 克隆并修改 var base *Config - if cur == nil { + if oldCfg == nil { base = NewConfig() } else { - base = CloneConfigShallow(cur) + base = CloneConfigShallow(oldCfg) } if err := mutator(base); err != nil { updateErr = err @@ -757,6 +788,7 @@ func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist return nil, updateErr } + fireAfterUpdate(oldCfg, newCfg) return newCfg, nil } diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 7d8e3ab..7ce14d4 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -86,6 +86,9 @@ func (e *Executor) Execute( var commands []string var logs string + // 记录阶段开始时间 + stageStartTime := getTimeNow() + // 并行阶段处理 if stageCfg.IsParallel() { e.logger.WithField("stage_index", i).Debug("executing parallel stages") @@ -107,7 +110,7 @@ func (e *Executor) Execute( Commands: commands, Logs: logs, } - result.StartedAt = getTimeNow() + result.StartedAt = stageStartTime if err != nil { result.Status = StageStatusFailed @@ -144,8 +147,19 @@ func (e *Executor) Execute( }).Debug("stage completed") } - // 全部成功：执行延迟删除 - e.deleteMarkedFiles(files) + // 全部成功：执行延迟删除，并让最终结果反映磁盘上的文件 + // 如果上下文已取消，跳过删除以保留所有文件 + if ctx.Ctx.Err() != nil { + e.logger.Warn("pipeline cancelled after completion, skipping file cleanup") + if len(results) > 0 { + results[len(results)-1].OutputFiles = files + } + return results, nil + } + keptFiles := e.deleteMarkedFiles(files) + if len(results) > 0 { + results[len(results)-1].OutputFiles = keptFiles + } return results, nil } @@ -183,7 +197,7 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File dirs := map[string]bool{} for _, f := range allFiles { ext := strings.ToLower(filepath.Ext(f.Path)) - if ext == ".flv" || ext == ".mp4" || ext == ".mkv" { + if ext == ".flv" || ext == ".mp4" || ext == ".mkv" || ext == ".ts" { dir := filepath.Dir(f.Path) base := strings.TrimSuffix(filepath.Base(f.Path), ext) videoBaseNames[filepath.Join(dir, base)] = true @@ -193,13 +207,21 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File // 收集 Pipeline 中已知的 .ass 文件路径 knownAssFiles := map[string]bool{} + // 同时收集 allFiles 中的 .ass 文件路径（用于限定扫描范围） + allAssFiles := map[string]bool{} for _, f := range keptFiles { if strings.ToLower(filepath.Ext(f.Path)) == ".ass" { knownAssFiles[f.Path] = true } } + for _, f := range allFiles { + if strings.ToLower(filepath.Ext(f.Path)) == ".ass" { + allAssFiles[f.Path] = true + } + } - // 扫描目录中的 .ass 文件（包括未进入 Pipeline 的） + // 扫描目录中的 .ass 文件 + // 仅处理本任务关联的 .ass 文件，避免误删其他任务或用户放置的字幕 for dir := range dirs { entries, err := os.ReadDir(dir) if err != nil { @@ -216,12 +238,15 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File if knownAssFiles[assPath] { continue // 已在 Pipeline 中处理 } + if !allAssFiles[assPath] { + continue // 非本任务关联的 .ass 文件，跳过 + } // 检查磁盘上是否有同名视频文件 base := strings.TrimSuffix(entry.Name(), ".ass") key := filepath.Join(dir, base) - // 检查磁盘上实际存在的视频文件 + // 检查磁盘上实际存在的视频文件（含 .ts 格式） hasVideo := false - for _, ext := range []string{".flv", ".mp4", ".mkv"} { + for _, ext := range []string{".flv", ".mp4", ".mkv", ".ts"} { if _, err := os.Stat(key + ext); err == nil { hasVideo = true break @@ -230,10 +255,10 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File if !hasVideo { if err := os.Remove(assPath); err != nil { if !os.IsNotExist(err) { - e.logger.Warnf("pipeline cleanup: 删除磁盘孤立 .ass 文件失败 %s: %v", assPath, err) + e.logger.Warnf("pipeline cleanup: 删除孤立 .ass 文件失败 %s: %v", assPath, err) } } else { - e.logger.Infof("pipeline cleanup: 已删除磁盘孤立 .ass 文件 %s（无关联视频）", assPath) + e.logger.Infof("pipeline cleanup: 已删除孤立 .ass 文件 %s（无关联视频）", assPath) } } } diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index da7880d..b7a7371 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -342,6 +342,7 @@ func (m *Manager) RetryTask(taskID int64) error { task.CompletedAt = nil task.ErrorMessage = "" task.CurrentStage = 0 + task.CurrentFiles = task.InitialFiles // 重置为初始文件，从头开始重试 task.StageResults = nil task.Progress = 0 diff --git a/src/pipeline/stages/burn_subtitles.go b/src/pipeline/stages/burn_subtitles.go index fa2f8db..5fce8f6 100644 --- a/src/pipeline/stages/burn_subtitles.go +++ b/src/pipeline/stages/burn_subtitles.go @@ -83,6 +83,13 @@ func (s *BurnSubtitlesStage) Execute(ctx *pipeline.PipelineContext, input []pipe continue } + // 跳过已标记待删除的文件（如 ConvertMp4 阶段标记的原始 FLV）， + // 但仍将其透传到 output，保证 Executor 的延迟删除机制正常工作。 + if file.Deletable { + output = append(output, file) + continue + } + // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) diff --git a/src/pipeline/stages/convert_mp4.go b/src/pipeline/stages/convert_mp4.go index 53daedb..73457e4 100644 --- a/src/pipeline/stages/convert_mp4.go +++ b/src/pipeline/stages/convert_mp4.go @@ -120,6 +120,12 @@ func (s *ConvertMp4Stage) Execute(ctx *pipeline.PipelineContext, input []pipelin return nil, fmt.Errorf("failed to create stdout pipe: %w", err) } + stderr, err := cmd.StderrPipe() + if err != nil { + s.logs += fmt.Sprintf("创建错误管道失败: %s\n", err.Error()) + return nil, fmt.Errorf("failed to create stderr pipe: %w", err) + } + if err := cmd.Start(); err != nil { s.logs += fmt.Sprintf("启动 ffmpeg 失败: %s\n", err.Error()) return nil, fmt.Errorf("failed to start ffmpeg: %w", err) @@ -130,10 +136,14 @@ func (s *ConvertMp4Stage) Execute(ctx *pipeline.PipelineContext, input []pipelin s.parseProgress(goCtx, stdout, duration) }) + // 读取 stderr 防止缓冲区满导致 ffmpeg 挂起 + stderrBytes, _ := io.ReadAll(stderr) + // 等待命令完成 if err := cmd.Wait(); err != nil { os.Remove(tempFile) s.logs += fmt.Sprintf("ffmpeg 转换失败: %s - %s\n", file.Path, err.Error()) + s.logs += fmt.Sprintf("ffmpeg stderr: %s\n", string(stderrBytes)) return nil, fmt.Errorf("ffmpeg conversion failed for %s: %w", file.Path, err) } diff --git a/src/pipeline/stages/custom_command.go b/src/pipeline/stages/custom_command.go index c5ddc10..50b7186 100644 --- a/src/pipeline/stages/custom_command.go +++ b/src/pipeline/stages/custom_command.go @@ -51,6 +51,13 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe var output []pipeline.FileInfo for _, file := range input { + // 跳过已标记待删除的文件（如 ConvertMp4/BurnSubtitles 阶段标记的中间产物）， + // 避免对本应删除的文件重复执行命令，但仍将其透传到 output。 + if file.Deletable { + output = append(output, file) + continue + } + // 渲染命令模板 cmdStr, err := s.renderCommand(ctx, file) if err != nil { diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index 5547d78..abc4e71 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -52,6 +52,12 @@ func (s *ExtractCoverStage) Execute(ctx *pipeline.PipelineContext, input []pipel continue } + // 跳过已标记待删除的文件（如 ConvertMp4 阶段标记的原始 FLV）， + // 避免重复提取封面。 + if file.Deletable { + continue + } + // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) @@ -134,8 +140,8 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 获取 OpenList 管理器 mgr := openlist.GetGlobalManager() if mgr == nil { - s.logs = "OpenList 管理器未初始化，跳过上传\n" - return input, nil + s.logs = "OpenList 管理器未初始化\n" + return input, fmt.Errorf("OpenList 管理器未初始化") } // 获取配置并创建客户端 @@ -149,8 +155,11 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 如果 token 变化了（通过用户名密码登录获取了新 token），自动写回配置 if newToken := client.GetCurrentToken(); newToken != "" && newToken != config.OpenList.Token { - config.OpenList.Token = newToken - if err := config.Marshal(); err != nil { + _, err := configs.UpdateWithRetry(func(c *configs.Config) error { + c.OpenList.Token = newToken + return nil + }, 3, 10*time.Millisecond) + if err != nil { ctx.Logger.Warnf("保存 OpenList token 到配置文件失败: %v", err) } else { ctx.Logger.Info("OpenList token 已自动更新到配置文件") @@ -160,16 +169,19 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli var output []pipeline.FileInfo var uploadFailed bool - // 清除所有输入文件的 Deletable 标记（防止数据库中残留的旧标记影响本次判断） - for i := range input { - input[i].Deletable = false - } - - // 只上传第一个视频文件和封面文件，跳过中间产物 - // Pipeline 阶段输出顺序保证：最终视频在最前面，中间文件在后面 - videoUploaded := false + // 上传所有非 Deletable 的视频文件和封面文件 + // Deletable 检查已能正确区分中间产物（如 ConvertMp4 标记的原始 FLV） + // 与最终视频（如录播姬生成的多分段 _PART 文件），无需额外限制 uploadedIndices := map[int]bool{} // 记录实际上传的文件在 output 中的索引 + // 统计待上传的视频文件数量，用于判断是否需要在路径中包含文件名以避免冲突 + nonDeletableVideoCount := 0 + for _, file := range input { + if file.Type == pipeline.FileTypeVideo && !file.Deletable { + nonDeletableVideoCount++ + } + } + for _, file := range input { // 跳过标记为待删除的中间文件（由其他阶段产出，不应上传） if file.Deletable { @@ -183,23 +195,17 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli continue } - // 视频文件：只上传第一个（最终视频），跳过后续中间视频 - if file.Type == pipeline.FileTypeVideo { - if videoUploaded { - output = append(output, file) - continue - } - videoUploaded = true - } - // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) + output = append(output, file) + uploadFailed = true continue } // 渲染目标路径 - targetPath := s.renderTargetPath(ctx, file) + // 多视频场景（如录播姬 _PART 分段）时，若模板不含 .FileName 则自动追加，确保路径唯一 + targetPath := s.renderTargetPath(ctx, file, nonDeletableVideoCount > 1) if targetPath == "" { s.logs += fmt.Sprintf("无法生成目标路径: %s\n", file.Path) output = append(output, file) @@ -293,7 +299,8 @@ func (s *CloudUploadStage) matchFileType(fileType pipeline.FileType) bool { } // renderTargetPath 渲染目标路径 -func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo) string { +// multiVideo 为 true 时，若模板不含 .FileName 则自动追加，确保多分段文件路径唯一 +func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo, multiVideo bool) string { if s.pathTemplate == "" { // 默认路径：/录播归档/{平台}/{主播名}/{文件名} return fmt.Sprintf("/录播归档/%s/%s/%s", @@ -303,6 +310,18 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file ) } + // 多视频场景下，若模板不含 .FileName 则自动追加以避免路径冲突 + templateStr := s.pathTemplate + if multiVideo && !strings.Contains(templateStr, ".FileName") { + // 在模板末尾（扩展名之前）插入文件名 + // 例如: /path/房间名-日期.{{ .Ext }} → /path/房间名-日期_{{ .FileName }}.{{ .Ext }} + templateStr = strings.TrimSuffix(templateStr, ".{{ .Ext }}") + "_{{ .FileName }}.{{ .Ext }}" + if !strings.Contains(templateStr, ".FileName") { + // 兜底：直接在末尾追加 + templateStr = s.pathTemplate + "/{{ .FileName }}" + } + } + // 获取扩展名 ext := filepath.Ext(file.Path) if len(ext) > 0 && ext[0] == '.' { @@ -330,7 +349,7 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file return t.Format(format) }, "now": func() time.Time { - return time.Now() + return ctx.RecordInfo.StartTime }, "trimSuffix": func(suffix, s string) string { return strings.TrimSuffix(s, suffix) @@ -348,43 +367,22 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file }, } - tmpl, err := template.New("path").Funcs(funcMap).Parse(s.pathTemplate) + tmpl, err := template.New("path").Funcs(funcMap).Parse(templateStr) if err != nil { - // 模板解析失败，回退到简单替换 - return s.fallbackRender(data) + // 模板语法错误，返回空让调用方跳过上传，避免上传到错误路径 + ctx.Logger.Errorf("上传路径模板解析失败: %v", err) + return "" } var buf bytes.Buffer if err := tmpl.Execute(&buf, data); err != nil { - // 模板执行失败，回退到简单替换 - return s.fallbackRender(data) + ctx.Logger.Errorf("上传路径模板执行失败: %v", err) + return "" } return buf.String() } -// fallbackRender 简单字符串替换（模板引擎失败时的回退方案） -func (s *CloudUploadStage) fallbackRender(data struct { - Platform string - HostName string - RoomName string - FileName string - Ext string -}) string { - path := s.pathTemplate - path = strings.ReplaceAll(path, "{{ .Platform }}", data.Platform) - path = strings.ReplaceAll(path, "{{.Platform}}", data.Platform) - path = strings.ReplaceAll(path, "{{ .HostName }}", data.HostName) - path = strings.ReplaceAll(path, "{{.HostName}}", data.HostName) - path = strings.ReplaceAll(path, "{{ .RoomName }}", data.RoomName) - path = strings.ReplaceAll(path, "{{.RoomName}}", data.RoomName) - path = strings.ReplaceAll(path, "{{ .FileName }}", data.FileName) - path = strings.ReplaceAll(path, "{{.FileName}}", data.FileName) - path = strings.ReplaceAll(path, "{{ .Ext }}", data.Ext) - path = strings.ReplaceAll(path, "{{.Ext}}", data.Ext) - return path -} - func (s *CloudUploadStage) GetCommands() []string { return s.commands } diff --git a/src/pkg/openlist/client.go b/src/pkg/openlist/client.go index 0c6ab14..89bf598 100644 --- a/src/pkg/openlist/client.go +++ b/src/pkg/openlist/client.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" "os" "strings" + "sync" ) // encodePath 对路径的每个段分别编码，保留 / 分隔符 @@ -24,6 +26,7 @@ func encodePath(path string) string { // Client OpenList API 客户端 type Client struct { baseURL string + mu sync.RWMutex // 保护 token/username/password 的并发访问 token string username string // 用于 token 刷新 password string // 用于 token 刷新 @@ -43,40 +46,75 @@ func NewClient(baseURL, token string) *Client { // SetToken 设置 API Token func (c *Client) SetToken(token string) { + c.mu.Lock() + defer c.mu.Unlock() c.token = token } -// GetToken 获取当前 API Token +// GetCurrentToken 获取当前 API Token func (c *Client) GetCurrentToken() string { + c.mu.RLock() + defer c.mu.RUnlock() return c.token } // SetCredentials 设置用户名密码（用于 token 自动刷新） func (c *Client) SetCredentials(username, password string) { + c.mu.Lock() + defer c.mu.Unlock() c.username = username c.password = password } +// getTokenForRequest 获取用于请求的 token 副本（内部使用） +func (c *Client) getTokenForRequest() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.token +} + // refreshToken 刷新 token func (c *Client) refreshToken(ctx context.Context) error { - if c.username == "" || c.password == "" { + c.mu.RLock() + username := c.username + password := c.password + c.mu.RUnlock() + + if username == "" || password == "" { return fmt.Errorf("无法刷新 token: 未配置用户名密码") } - newToken, err := c.GetToken(ctx, c.username, c.password) + newToken, err := c.GetToken(ctx, username, password) if err != nil { return fmt.Errorf("刷新 token 失败: %w", err) } - c.token = newToken + c.SetToken(newToken) return nil } -// isAuthError 判断是否为认证错误（HTTP 401/403） +// APIError 表示 OpenList API 返回的结构化错误 +// 用于在 isAuthError 中识别认证失败（code 401/403） +type APIError struct { + Code int + Message string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("API 错误 (code %d): %s", e.Code, e.Message) +} + +// isAuthError 判断是否为认证错误 +// 匹配 HTTP 状态码 401/403 或 OpenList JSON code 401/403 func isAuthError(err error) bool { if err == nil { return false } + // 检查结构化 API 错误 + var apiErr *APIError + if errors.As(err, &apiErr) { + return apiErr.Code == 401 || apiErr.Code == 403 + } + // 兜底：检查错误文本中的 HTTP 状态码 s := err.Error() - // 匹配 "HTTP 401" 或 "HTTP 403" 格式，避免误匹配其他包含 401/403 的字符串 return strings.Contains(s, "HTTP 401") || strings.Contains(s, "HTTP 403") } @@ -123,7 +161,7 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP return fmt.Errorf("创建请求失败: %w", err) } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Length", fmt.Sprintf("%d", totalSize)) // 对路径的每个段分别编码，保留 / 分隔符 @@ -148,12 +186,14 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP Message string `json:"message"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - // 响应不是 JSON 格式，可能是空响应（某些 API 成功时返回空 body） - // 由于 HTTP 状态码是 200，视为成功 - return nil + // 仅空响应（io.EOF）视为成功，某些 API 成功时返回空 body + if err == io.EOF { + return nil + } + return fmt.Errorf("解析上传响应失败: %w", err) } if result.Code != 200 { - return fmt.Errorf("上传失败: %s", result.Message) + return fmt.Errorf("上传失败: %w", &APIError{Code: result.Code, Message: result.Message}) } return nil @@ -176,7 +216,7 @@ func (c *Client) ListStorages(ctx context.Context) ([]StorageInfo, error) { if err != nil { return err } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) resp, err := c.httpClient.Do(req) if err != nil { @@ -197,7 +237,7 @@ func (c *Client) ListStorages(ctx context.Context) ([]StorageInfo, error) { } if result.Code != 200 { - return fmt.Errorf("API 错误: %s", result.Message) + return fmt.Errorf("API 错误: %w", &APIError{Code: result.Code, Message: result.Message}) } storages = result.Data.Content @@ -218,7 +258,7 @@ func (c *Client) CheckStorageHealth(ctx context.Context, storageName string) err if err != nil { return err } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) req.Header.Set("Content-Type", "application/json") resp, err := c.httpClient.Do(req) @@ -237,7 +277,7 @@ func (c *Client) CheckStorageHealth(ctx context.Context, storageName string) err } if result.Code != 200 { - return fmt.Errorf("存储不可用: %s", result.Message) + return fmt.Errorf("存储不可用: %w", &APIError{Code: result.Code, Message: result.Message}) } return nil @@ -276,7 +316,7 @@ func (c *Client) GetToken(ctx context.Context, username, password string) (strin } if result.Code != 200 { - return "", fmt.Errorf("登录失败: %s", result.Message) + return "", fmt.Errorf("登录失败: %w", &APIError{Code: result.Code, Message: result.Message}) } return result.Data.Token, nil @@ -293,7 +333,7 @@ func (c *Client) Mkdir(ctx context.Context, remotePath string) error { if err != nil { return err } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) req.Header.Set("Content-Type", "application/json") resp, err := c.httpClient.Do(req) @@ -313,7 +353,7 @@ func (c *Client) Mkdir(ctx context.Context, remotePath string) error { // 目录已存在也算成功 if result.Code != 200 && result.Message != "file exists" { - return fmt.Errorf("创建目录失败: %s", result.Message) + return fmt.Errorf("创建目录失败: %w", &APIError{Code: result.Code, Message: result.Message}) } return nil diff --git a/src/pkg/openlist/manager.go b/src/pkg/openlist/manager.go index 1d4e642..995af86 100644 --- a/src/pkg/openlist/manager.go +++ b/src/pkg/openlist/manager.go @@ -227,9 +227,11 @@ func (m *Manager) saveInitialCredentials(logFile string) { } // 保存到配置 - config.OpenList.Username = "admin" - config.OpenList.Password = initialPassword - if err := config.Marshal(); err != nil { + if _, err := configs.UpdateWithRetry(func(c *configs.Config) error { + c.OpenList.Username = "admin" + c.OpenList.Password = initialPassword + return nil + }, 3, 10*time.Millisecond); err != nil { logrus.WithError(err).Warn("保存 OpenList 初始密码到配置文件失败") } else { logrus.Info("OpenList 首次启动，初始密码已保存到配置文件") diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index 4014ba9..dfc9753 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -792,7 +792,7 @@ func (r *recorder) tryRecord(ctx context.Context) { for _, stage := range pipelineConfig.Stages { stageNames = append(stageNames, stage.Name) } - r.getLogger().Infof("pipeline task enqueued: %d files, stages: %s", len(outputFiles), stageNames) + r.getLogger().Infof("pipeline task enqueued: %d files, stages: %v", len(outputFiles), stageNames) } } } diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 4ad5712..430e623 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -44,8 +44,8 @@ const UPLOAD_PRESETS: PresetTemplate[] = [ }, { name: '简洁归档', - description: '按平台/主播归档，文件名带日期', - template: '/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }}', + description: '按平台/主播归档，文件名带日期时间', + template: '/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02 15-04-05" }}.{{ .Ext }}', }, { name: '按房间归档', @@ -95,7 +95,7 @@ const MOCK_DATA: MockStream[] = [ const renderUploadTemplate = (template: string, stream: MockStream, room: MockStream['rooms'][0]): string => { const fileName = `[${room.date} ${room.time}][${stream.hostName}][${room.roomName}].flv`; let path = template; - path = path.replace(/\{\{ ?\.Platform ?\}\}/g, stream.platform); + path = path.replace(/\{\{ ?\.Platform ?\}\}/g, stream.platformCN); path = path.replace(/\{\{ ?\.HostName ?\}\}/g, stream.hostName); path = path.replace(/\{\{ ?\.RoomName ?\}\}/g, room.roomName); path = path.replace(/\{\{ ?\.FileName ?\}\}/g, fileName); @@ -291,6 +291,27 @@ interface CloudUploadSettingsProps { const CloudUploadSettings: React.FC = ({ config, form }) => { const isEnabled = config.on_record_finished?.cloud_upload?.enable; + // 订阅表单中影响文件处理预览的字段，使预览实时反映用户编辑 + const watchedOrf = Form.useWatch('on_record_finished', form); + const watchedCloudUpload = Form.useWatch(['on_record_finished', 'cloud_upload'], form); + + // 将表单实时值合并到 config 副本中，供 FileProcessingPreview 使用 + // Form.useWatch 返回 undefined 时表示字段未被编辑，此时保留 config 中的原始值 + const liveConfig = useMemo(() => { + if (!watchedOrf && !watchedCloudUpload) return config; + return { + ...config, + on_record_finished: { + ...config.on_record_finished, + ...watchedOrf, + cloud_upload: { + ...config.on_record_finished?.cloud_upload, + ...watchedCloudUpload, + }, + }, + }; + }, [config, watchedOrf, watchedCloudUpload]); + // 互斥逻辑：开启一个时关闭另一个 const handleDeleteAfterChange = (checked: boolean) => { if (checked && form) { @@ -319,7 +340,7 @@ const CloudUploadSettings: React.FC = ({ config, form <> 录制结束后自动把视频传到网盘。需要先在{' '} @@ -333,8 +354,8 @@ const CloudUploadSettings: React.FC = ({ config, form style={{ marginBottom: 16 }} /> - {/* 文件处理预览 */} - + {/* 文件处理预览 - 使用表单实时值，无需保存即可反映用户编辑 */} + @@ -369,7 +390,7 @@ const CloudUploadSettings: React.FC = ({ config, form presets={UPLOAD_PRESETS} mockData={MOCK_DATA} renderTemplate={renderUploadTemplate} - placeholder="/录播归档/{{ .Platform }}/{{ .HostName }}/{{ now | date '2006-01-02' }}/{{ .FileName }}" + placeholder={'/录播归档/{{ .Platform }}/{{ .HostName }}/{{ now | date "2006-01-02" }}/{{ .FileName }}'} showTreePreview={true} width={500} /> From 388e1729cf8056756c426e00ffbbfa3e2cc7d918 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:00:36 +0800 Subject: [PATCH 02/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D=20Del?= =?UTF-8?q?etable=20=E8=AF=AD=E4=B9=89=E9=87=8D=E8=BD=BD=E5=92=8C=E5=A4=9A?= =?UTF-8?q?=E5=A4=84=E8=BE=B9=E7=95=8C=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletable 语义分离： - CloudUploadStage deleteAfter 模式改用 Metadata[uploaded] 标记，不再设 Deletable - 修复 delete_after_upload 后自定义命令跳过最终成品的回归 - Executor deleteMarkedFiles 同时检查 Deletable 和 Metadata[uploaded] - 仅 deleteAll 模式下执行孤立 .ass 清理，deleteAfter 不连带删弹幕 路径冲突检测： - 移除 multiVideo 模板追加逻辑，改用渲染后 usedPaths 冲突检测 - 冲突检测覆盖所有待上传文件（视频+封面），不只是视频 - 删除无用的 fallbackRender 函数 Executor 修复： - 上下文取消时跳过 deleteMarkedFiles，保留所有文件 - 删除失败的文件保留在 kept 中，CurrentFiles 与磁盘一致 - StartedAt 在阶段执行前记录 OpenList 修复： - 移除 ListStorages 预验证，token 刷新由 withRetry 统一处理 - 移除 io.EOF 空响应特殊处理，要求有效 JSON 响应 - saveInitialCredentials 验证密码有效性后再保存 其他： - 云上传关闭时停止 OpenList 进程并清除引用 - configs.GetCurrentConfig() 判空防护 - %s 改为 %v 格式化 []string --- src/cmd/bililive/bililive.go | 14 ++++- src/pipeline/executor.go | 30 +++++++-- src/pipeline/stages/extract_cover.go | 92 +++++++++++++++++++--------- src/pkg/openlist/client.go | 4 -- src/pkg/openlist/manager.go | 26 ++++---- 5 files changed, 113 insertions(+), 53 deletions(-) diff --git a/src/cmd/bililive/bililive.go b/src/cmd/bililive/bililive.go index 669160c..a5e616d 100644 --- a/src/cmd/bililive/bililive.go +++ b/src/cmd/bililive/bililive.go @@ -371,9 +371,21 @@ func main() { if newCfg == nil { return } - // 检测云上传开关是否从关变为开 wasEnabled := oldCfg != nil && oldCfg.OnRecordFinished.CloudUpload.Enable isEnabled := newCfg.OnRecordFinished.CloudUpload.Enable + + // 关闭云上传：停止 OpenList 进程 + if wasEnabled && !isEnabled { + if mgr := openlist.GetGlobalManager(); mgr != nil { + logger.Info("检测到云上传已关闭，正在停止 OpenList...") + mgr.Stop() + servers.SetOpenListManager(nil) + openlistManager = nil + } + return + } + + // 开启云上传：启动 OpenList 进程 if wasEnabled || !isEnabled { return } diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 7ce14d4..ec3e354 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -164,14 +164,32 @@ func (e *Executor) Execute( return results, nil } -// deleteMarkedFiles 删除标记为 Deletable 的文件，返回保留的文件列表 +// deleteMarkedFiles 删除标记为 Deletable 或 Metadata["uploaded"] 的文件，返回保留的文件列表 func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { var kept []FileInfo + deleteAll := false // 标记是否为 deleteAll 模式 + for _, f := range files { - if f.Deletable { + // 检查是否为 deleteAll 模式（CloudUploadStage 标记） + if f.Metadata != nil { + if da, ok := f.Metadata["delete_all"].(bool); ok && da { + deleteAll = true + } + } + + // 判断是否应删除：Deletable 标记 或 CloudUploadStage 标记的已上传文件 + shouldDelete := f.Deletable + if !shouldDelete && f.Metadata != nil { + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + shouldDelete = true + } + } + + if shouldDelete { if err := os.Remove(f.Path); err != nil { if !os.IsNotExist(err) { e.logger.Warnf("pipeline cleanup: 删除文件失败 %s: %v", f.Path, err) + kept = append(kept, f) // 删除失败，文件仍在磁盘，保留记录 } } else { e.logger.Infof("pipeline cleanup: 已删除 %s", f.Path) @@ -181,9 +199,11 @@ func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { } } - // 兜底逻辑：清理无关联视频文件的 .ass 文件 - // 传入原始 files 列表，确保即使视频被删除也能扫描到对应目录 - kept = e.cleanupOrphanedAssFiles(files, kept) + // 仅在 deleteAll 模式下清理孤立 .ass 文件 + // deleteAfter 模式只删除已上传文件，不应连带清理弹幕 + if deleteAll { + kept = e.cleanupOrphanedAssFiles(files, kept) + } return kept } diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index abc4e71..f8224b3 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -146,6 +146,10 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 获取配置并创建客户端 config := configs.GetCurrentConfig() + if config == nil { + s.logs = "配置未初始化\n" + return input, fmt.Errorf("配置未初始化") + } client, err := mgr.GetClient(ctx.Ctx, config.OpenList.Token, config.OpenList.Username, config.OpenList.Password) if err != nil { @@ -168,20 +172,13 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli var output []pipeline.FileInfo var uploadFailed bool + usedPaths := map[string]bool{} // 检测远程路径冲突 // 上传所有非 Deletable 的视频文件和封面文件 // Deletable 检查已能正确区分中间产物（如 ConvertMp4 标记的原始 FLV） // 与最终视频（如录播姬生成的多分段 _PART 文件），无需额外限制 uploadedIndices := map[int]bool{} // 记录实际上传的文件在 output 中的索引 - // 统计待上传的视频文件数量，用于判断是否需要在路径中包含文件名以避免冲突 - nonDeletableVideoCount := 0 - for _, file := range input { - if file.Type == pipeline.FileTypeVideo && !file.Deletable { - nonDeletableVideoCount++ - } - } - for _, file := range input { // 跳过标记为待删除的中间文件（由其他阶段产出，不应上传） if file.Deletable { @@ -204,8 +201,7 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } // 渲染目标路径 - // 多视频场景（如录播姬 _PART 分段）时，若模板不含 .FileName 则自动追加，确保路径唯一 - targetPath := s.renderTargetPath(ctx, file, nonDeletableVideoCount > 1) + targetPath := s.renderTargetPath(ctx, file) if targetPath == "" { s.logs += fmt.Sprintf("无法生成目标路径: %s\n", file.Path) output = append(output, file) @@ -213,6 +209,17 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli continue } + // 检测远程路径冲突（多分段视频或视频+封面使用相同模板时可能产生同路径） + // 冲突时自动追加原始文件名以区分 + if usedPaths[targetPath] { + dir := filepath.Dir(targetPath) + ext := filepath.Ext(targetPath) + base := strings.TrimSuffix(filepath.Base(targetPath), ext) + targetPath = filepath.Join(dir, base+"_"+filepath.Base(file.Path)) + ctx.Logger.Infof("检测到远程路径冲突，自动追加文件名: %s", targetPath) + } + usedPaths[targetPath] = true + ctx.Logger.Infof("上传文件: %s -> %s", file.Path, targetPath) s.commands = append(s.commands, fmt.Sprintf("upload %s to %s/%s", file.Path, s.storageName, targetPath)) @@ -258,20 +265,62 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 全部上传成功后，根据配置标记文件为可删除（由 Executor 在管道全部成功后统一删除） // 任意一个上传失败，则不标记任何文件，确保本地文件全部保留 + // 注意：不使用 Deletable 标记，避免后续阶段（如 CustomCommand）误跳过最终成品 + // 改用 Metadata["uploaded"] 标记，由 Executor 在清理时读取 if !uploadFailed && s.uploadTiming != "immediate" { if s.deleteAllAfter { - // 删除全部文件（含中间产物） + // 删除全部模式：查找视频文件关联的 .ass 字幕文件并加入 output + // （BurnSubtitlesStage 仅在 burn_delete_ass=true 时才将 .ass 加入 output， + // 但 delete_all 意图是删除所有文件，应包含字幕） + assAdded := map[string]bool{} + for _, f := range output { + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { + continue + } + assPath := strings.TrimSuffix(f.Path, ext) + ".ass" + if assAdded[assPath] { + continue + } + if _, err := os.Stat(assPath); err == nil { + // 检查 output 中是否已存在 + alreadyInOutput := false + for _, of := range output { + if of.Path == assPath { + alreadyInOutput = true + break + } + } + if !alreadyInOutput { + output = append(output, pipeline.FileInfo{ + Path: assPath, + Type: pipeline.FileTypeOther, + }) + assAdded[assPath] = true + ctx.Logger.Infof("delete_all 模式：关联字幕文件 %s", assPath) + } + } + } + + // 删除全部文件（含中间产物）：用 Deletable 标记 for i := range output { output[i].Deletable = true + if output[i].Metadata == nil { + output[i].Metadata = map[string]any{} + } + output[i].Metadata["delete_all"] = true } s.logs += fmt.Sprintf("全部上传成功，已标记 %d 个文件待删除（含中间产物）\n", len(output)) ctx.Logger.Infof("全部上传成功，已标记 %d 个文件待删除（含中间产物）", len(output)) } else if s.deleteAfter { - // 仅删除已上传的文件 + // 仅删除已上传的文件：用 Metadata["uploaded"] 标记，不设 Deletable count := 0 for i := range output { if uploadedIndices[i] { - output[i].Deletable = true + if output[i].Metadata == nil { + output[i].Metadata = map[string]any{} + } + output[i].Metadata["uploaded"] = true count++ } } @@ -299,8 +348,7 @@ func (s *CloudUploadStage) matchFileType(fileType pipeline.FileType) bool { } // renderTargetPath 渲染目标路径 -// multiVideo 为 true 时，若模板不含 .FileName 则自动追加，确保多分段文件路径唯一 -func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo, multiVideo bool) string { +func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo) string { if s.pathTemplate == "" { // 默认路径：/录播归档/{平台}/{主播名}/{文件名} return fmt.Sprintf("/录播归档/%s/%s/%s", @@ -310,18 +358,6 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file ) } - // 多视频场景下，若模板不含 .FileName 则自动追加以避免路径冲突 - templateStr := s.pathTemplate - if multiVideo && !strings.Contains(templateStr, ".FileName") { - // 在模板末尾（扩展名之前）插入文件名 - // 例如: /path/房间名-日期.{{ .Ext }} → /path/房间名-日期_{{ .FileName }}.{{ .Ext }} - templateStr = strings.TrimSuffix(templateStr, ".{{ .Ext }}") + "_{{ .FileName }}.{{ .Ext }}" - if !strings.Contains(templateStr, ".FileName") { - // 兜底：直接在末尾追加 - templateStr = s.pathTemplate + "/{{ .FileName }}" - } - } - // 获取扩展名 ext := filepath.Ext(file.Path) if len(ext) > 0 && ext[0] == '.' { @@ -367,7 +403,7 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file }, } - tmpl, err := template.New("path").Funcs(funcMap).Parse(templateStr) + tmpl, err := template.New("path").Funcs(funcMap).Parse(s.pathTemplate) if err != nil { // 模板语法错误，返回空让调用方跳过上传，避免上传到错误路径 ctx.Logger.Errorf("上传路径模板解析失败: %v", err) diff --git a/src/pkg/openlist/client.go b/src/pkg/openlist/client.go index 89bf598..59f8f5d 100644 --- a/src/pkg/openlist/client.go +++ b/src/pkg/openlist/client.go @@ -186,10 +186,6 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP Message string `json:"message"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - // 仅空响应（io.EOF）视为成功，某些 API 成功时返回空 body - if err == io.EOF { - return nil - } return fmt.Errorf("解析上传响应失败: %w", err) } if result.Code != 200 { diff --git a/src/pkg/openlist/manager.go b/src/pkg/openlist/manager.go index 995af86..e6159e6 100644 --- a/src/pkg/openlist/manager.go +++ b/src/pkg/openlist/manager.go @@ -226,6 +226,14 @@ func (m *Manager) saveInitialCredentials(logFile string) { return } + // 验证密码有效性：日志是追加模式，旧密码可能已被用户修改 + // 只有实际能登录时才保存 + client := NewClient(fmt.Sprintf("http://127.0.0.1:%d", m.port), "") + if _, err := client.GetToken(context.Background(), "admin", initialPassword); err != nil { + logrus.WithError(err).Debug("初始密码验证失败，跳过保存（可能已被修改）") + return + } + // 保存到配置 if _, err := configs.UpdateWithRetry(func(c *configs.Config) error { c.OpenList.Username = "admin" @@ -324,6 +332,7 @@ func (m *Manager) GetDataPath() string { // GetClient 获取已认证的 API 客户端 // 如果提供了 token，直接使用；否则使用用户名密码登录获取 token +// token 有效性由 withRetry 在实际 API 调用时自动处理（401/403 时自动刷新） func (m *Manager) GetClient(ctx context.Context, token, username, password string) (*Client, error) { client := NewClient(m.apiEndpoint, "") @@ -332,23 +341,10 @@ func (m *Manager) GetClient(ctx context.Context, token, username, password strin client.SetCredentials(username, password) } - // 有 token 时，验证其有效性 + // 有 token 时直接使用，不预验证（避免需要管理员权限的 API 调用） + // withRetry 会在实际调用遇到 401/403 时自动用密码刷新 token if token != "" { client.SetToken(token) - // 尝试用 token 调用一个轻量 API 验证有效性 - if _, err := client.ListStorages(ctx); err != nil { - // token 无效，尝试用密码重新登录 - if username != "" && password != "" { - logrus.Warn("配置的 OpenList token 无效，尝试用用户名密码重新登录") - newToken, loginErr := client.GetToken(ctx, username, password) - if loginErr != nil { - return nil, fmt.Errorf("OpenList token 无效且登录失败: %w", loginErr) - } - client.SetToken(newToken) - return client, nil - } - return nil, fmt.Errorf("OpenList token 无效且未配置用户名密码") - } return client, nil } From b6b2fc6af6e3e7a8577cb69b81707bbcf6ab6276 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:19:54 +0800 Subject: [PATCH 03/52] =?UTF-8?q?fix(openlist):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=AF=86=E7=A0=81=E4=B8=8D=E5=86=99=E5=85=A5?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=92=8C=E5=89=8D=E7=AB=AF=E5=87=AD=E6=8D=AE?= =?UTF-8?q?=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - saveInitialCredentials 验证现有凭据有效性，无效时从日志读取真实密码 - 修复 config.yml 中占位值 root/root 导致真实密码永远不写入的问题 - OpenList 链接改为直接端口访问，备选代理路径 - 链接旁显示登录凭据方便复制 --- src/pkg/openlist/manager.go | 11 +++++++--- .../config-info/CloudUploadSettings.tsx | 21 +++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/pkg/openlist/manager.go b/src/pkg/openlist/manager.go index e6159e6..d2a437d 100644 --- a/src/pkg/openlist/manager.go +++ b/src/pkg/openlist/manager.go @@ -197,9 +197,14 @@ func (m *Manager) saveInitialCredentials(logFile string) { return } - // 如果已经配置了用户名密码，跳过 + // 如果已经配置了用户名密码，验证有效性 if config.OpenList.Username != "" && config.OpenList.Password != "" { - return + // 用现有凭据尝试登录，验证是否有效 + testClient := NewClient(m.apiEndpoint, "") + if _, err := testClient.GetToken(context.Background(), config.OpenList.Username, config.OpenList.Password); err == nil { + return // 凭据有效，无需更新 + } + // 凭据无效（如占位值 root/root），继续从日志读取真实密码 } // 读取日志文件查找初始密码 @@ -228,7 +233,7 @@ func (m *Manager) saveInitialCredentials(logFile string) { // 验证密码有效性：日志是追加模式，旧密码可能已被用户修改 // 只有实际能登录时才保存 - client := NewClient(fmt.Sprintf("http://127.0.0.1:%d", m.port), "") + client := NewClient(m.apiEndpoint, "") if _, err := client.GetToken(context.Background(), "admin", initialPassword); err != nil { logrus.WithError(err).Debug("初始密码验证失败，跳过保存（可能已被修改）") return diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 430e623..e205d13 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -340,13 +340,30 @@ const CloudUploadSettings: React.FC = ({ config, form <> 录制结束后自动把视频传到网盘。需要先在{' '} OpenList 管理页面 {' '} - 添加网盘。 + 添加网盘。{' '} + {config.openlist?.username && config.openlist?.password && ( + <> + + (登录凭据: {config.openlist.username} / {config.openlist.password}) + {' '} + + )} + + (如无法访问，尝试{' '} + + 通过代理访问 + ) + } type="info" From 769ac2d53fba51809bb985abeb1e319d11a724a3 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:09:57 +0800 Subject: [PATCH 04/52] =?UTF-8?q?feat(pipeline):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=BB=8E=E6=8C=87=E5=AE=9A=E9=98=B6=E6=AE=B5=E9=87=8D=E8=AF=95?= =?UTF-8?q?=EF=BC=8C=E6=97=A0=E9=9C=80=E5=85=A8=E9=87=8F=E9=87=8D=E8=B7=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 RetryTaskFromStage 方法，支持从 Pipeline 中任意已完成的阶段开始重试， 跳过前置阶段，解决上传失败需要重新转码的问题。 后端改动： - Executor.Execute 增加 startStage 参数，支持跳过指定阶段 - Manager 新增 RetryTaskFromStage 方法，校验前置阶段状态和输入文件 - executeTask 传递 startStage 并合并新旧 StageResults - API handler 支持可选的 stage_name body 参数（向后兼容） 前端改动： - 阶段列表中每个已完成/失败的阶段显示「从此阶段重试」按钮 - 任务级重试按钮支持 Completed 状态 - 补充 burn_subtitles 阶段标签 Co-Authored-By: Claude Opus 4.8 --- src/pipeline/executor.go | 10 +- src/pipeline/executor_test.go | 14 +- src/pipeline/manager.go | 121 +++++++++++++++++- src/servers/pipeline_handler.go | 26 +++- .../component/pipeline-task-list/index.tsx | 33 ++++- 5 files changed, 186 insertions(+), 18 deletions(-) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index ec3e354..19f0e4a 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -48,10 +48,12 @@ func (e *Executor) getFactory(name string) (StageFactory, bool) { } // Execute 执行管道 +// startStage: 从第几个启用的阶段开始执行（0=全部执行，用于重试指定阶段） func (e *Executor) Execute( ctx *PipelineContext, config *PipelineConfig, initialFiles []FileInfo, + startStage int, onProgress func(stageIndex int, stageName string, status StageStatus), ) ([]StageResult, error) { if config == nil || len(config.Stages) == 0 { @@ -76,6 +78,12 @@ func (e *Executor) Execute( continue } + // 跳过 startStage 之前的阶段（用于从指定阶段重试） + if stageIndex < startStage { + stageIndex++ + continue + } + // 记录开始 if onProgress != nil { onProgress(stageIndex, stageCfg.Name, StageStatusRunning) @@ -460,7 +468,7 @@ func (e *Executor) ExecuteAsync( bilisentry.GoWithContext(ctx.Ctx, func(goCtx context.Context) { // 更新上下文 ctx.Ctx = goCtx - results, err := e.Execute(ctx, config, initialFiles, onProgress) + results, err := e.Execute(ctx, config, initialFiles, 0, onProgress) if onComplete != nil { onComplete(results, err) } diff --git a/src/pipeline/executor_test.go b/src/pipeline/executor_test.go index b06d808..97f6de3 100644 --- a/src/pipeline/executor_test.go +++ b/src/pipeline/executor_test.go @@ -86,7 +86,7 @@ func TestExecute_AllStagesSucceed_AllDeletableFilesRemoved(t *testing.T) { results, err := executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil) // 验证：无错误 assert.NoError(t, err) @@ -142,7 +142,7 @@ func TestExecute_StageFailed_AllFilesPreserved(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil) // 验证：返回错误 assert.Error(t, err) @@ -199,7 +199,7 @@ func TestExecute_Cancelled_AllFilesPreserved(t *testing.T) { _, _ = executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil) // 验证：stage2 返回成功，但下一轮循环检测到 context 已取消 // 注意：stage2 本身成功了，所以 deleteMarkedFiles 会被调用 @@ -248,7 +248,7 @@ func TestExecute_PassthroughPreservesDeletable(t *testing.T) { results, err := executor.Execute(ctx, config, []FileInfo{ {Path: file1, Type: FileTypeVideo}, {Path: file2, Type: FileTypeOther, Deletable: true}, - }, nil) + }, 0, nil) assert.NoError(t, err) assert.Len(t, results, 1) @@ -304,7 +304,7 @@ func TestExecute_UploadCancelledDuringUpload_FilePreserved(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: localFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil) // 验证：返回错误 assert.Error(t, err) @@ -363,7 +363,7 @@ func TestExecute_UploadFailsMiddleOfFileList_NoneDeleted(t *testing.T) { {Path: file1, Type: FileTypeVideo}, {Path: file2, Type: FileTypeVideo}, {Path: file3, Type: FileTypeVideo}, - }, nil) + }, 0, nil) assert.NoError(t, err) @@ -423,7 +423,7 @@ func TestExecute_StaleDeletableFromDB_ClearedOnFailure(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: file1, Type: FileTypeVideo, Deletable: true}, // 旧标记 {Path: file2, Type: FileTypeVideo, Deletable: true}, // 旧标记 - }, nil) + }, 0, nil) assert.NoError(t, err) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index b7a7371..ff2c3d6 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -3,6 +3,7 @@ package pipeline import ( "context" "fmt" + "os" "sync" "time" @@ -201,11 +202,21 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { WorkDir: "", // 后续可以从配置获取 } - // 执行管道 + // 执行管道（从 startStage 开始，支持从指定阶段重试） + startStage := task.CurrentStage + + // 保留目标阶段之前的结果（用于合并） + var preservedResults []StageResult + if startStage > 0 && len(task.StageResults) >= startStage { + preservedResults = make([]StageResult, startStage) + copy(preservedResults, task.StageResults[:startStage]) + } + results, err := m.executor.Execute( pipelineCtx, task.PipelineConfig, task.CurrentFiles, + startStage, func(stageIndex int, stageName string, status StageStatus) { // 更新任务进度 task.CurrentStage = stageIndex @@ -217,8 +228,12 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { }, ) - // 保存阶段结果 - task.StageResults = results + // 合并：保留之前的结果 + 新执行的结果 + if preservedResults != nil { + task.StageResults = append(preservedResults, results...) + } else { + task.StageResults = results + } if err != nil { if ctx.Err() == context.Canceled { @@ -358,6 +373,106 @@ func (m *Manager) RetryTask(taskID int64) error { return nil } +// RetryTaskFromStage 从指定阶段开始重试任务 +// 允许 Failed/Cancelled/Completed 状态的任务 +// stageName: 要从哪个阶段开始重试（如 "cloud_upload"） +func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error { + task, err := m.store.GetTask(m.ctx, taskID) + if err != nil { + return err + } + + if !task.CanRetry { + return fmt.Errorf("task cannot be retried") + } + + // 允许 Failed/Cancelled/Completed 状态 + if task.Status != PipelineStatusFailed && + task.Status != PipelineStatusCancelled && + task.Status != PipelineStatusCompleted { + return fmt.Errorf("only failed, cancelled or completed tasks can be retried") + } + + // 找到目标阶段在启用阶段中的索引 + targetStageIndex := -1 + stageIndex := 0 + for _, stageCfg := range task.PipelineConfig.Stages { + if !stageCfg.IsEnabled() { + continue + } + if stageCfg.Name == stageName { + targetStageIndex = stageIndex + break + } + stageIndex++ + } + if targetStageIndex == -1 { + return fmt.Errorf("stage %s not found in pipeline", stageName) + } + + // 校验：目标阶段之前的所有阶段必须已完成 + if targetStageIndex > 0 { + for i := 0; i < targetStageIndex; i++ { + if i >= len(task.StageResults) { + return fmt.Errorf("stage %d has not been executed, cannot retry from stage %s", i, stageName) + } + if task.StageResults[i].Status != StageStatusCompleted { + return fmt.Errorf("stage %d (%s) is not completed (status: %s), cannot retry from stage %s", + i, task.StageResults[i].StageName, task.StageResults[i].Status, stageName) + } + } + } + + // 获取目标阶段的输入文件 + var inputFiles []FileInfo + if targetStageIndex < len(task.StageResults) { + // 从已有的 StageResult 中取输入文件 + inputFiles = task.StageResults[targetStageIndex].InputFiles + } else if targetStageIndex > 0 && targetStageIndex-1 < len(task.StageResults) { + // 使用上一阶段的输出文件 + inputFiles = task.StageResults[targetStageIndex-1].OutputFiles + } else if targetStageIndex == 0 { + // 从头开始，使用初始文件 + inputFiles = task.InitialFiles + } + + // 校验输入文件不为空 + if len(inputFiles) == 0 { + return fmt.Errorf("no input files available for stage %s, cannot retry", stageName) + } + + // 验证文件存在于磁盘 + for _, f := range inputFiles { + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + return fmt.Errorf("input file not found, cannot retry: %s", f.Path) + } + } + + // 重置任务状态 + task.Status = PipelineStatusPending + task.StartedAt = nil + task.CompletedAt = nil + task.ErrorMessage = "" + task.CurrentStage = targetStageIndex + task.CurrentFiles = inputFiles + // 保留目标阶段之前的结果，清除目标阶段及之后的结果 + if targetStageIndex < len(task.StageResults) { + task.StageResults = task.StageResults[:targetStageIndex] + } + task.Progress = (targetStageIndex * 100) / task.TotalStages + + if err := m.store.UpdateTask(m.ctx, task); err != nil { + return err + } + + m.broadcastTaskUpdate(task) + + // 立即尝试调度 + bilisentry.Go(func() { m.scheduleNextTasks() }) + + return nil +} + // GetTask 获取任务详情 func (m *Manager) GetTask(taskID int64) (*PipelineTask, error) { return m.store.GetTask(m.ctx, taskID) diff --git a/src/servers/pipeline_handler.go b/src/servers/pipeline_handler.go index 68f16ca..ea0e584 100644 --- a/src/servers/pipeline_handler.go +++ b/src/servers/pipeline_handler.go @@ -165,6 +165,8 @@ func makePipelineCancelTaskHandler(pm *pipeline.Manager) http.HandlerFunc { } // makePipelineRetryTaskHandler 重试任务 +// 支持可选的 JSON body: {"stage_name": "cloud_upload"} +// 无 body 或 stage_name 为空时全量重试 func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -174,9 +176,27 @@ func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc { return } - if err := pm.RetryTask(id); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + // 解析可选的 JSON body + var reqBody struct { + StageName string `json:"stage_name"` + } + if r.Body != nil { + // 尝试解析 body，失败不影响（向后兼容无 body 的情况） + json.NewDecoder(r.Body).Decode(&reqBody) + } + + if reqBody.StageName != "" { + // 从指定阶段重试 + if err := pm.RetryTaskFromStage(id, reqBody.StageName); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } else { + // 全量重试（向后兼容） + if err := pm.RetryTask(id); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } w.WriteHeader(http.StatusOK) diff --git a/src/webapp/src/component/pipeline-task-list/index.tsx b/src/webapp/src/component/pipeline-task-list/index.tsx index b05062d..db22b4b 100644 --- a/src/webapp/src/component/pipeline-task-list/index.tsx +++ b/src/webapp/src/component/pipeline-task-list/index.tsx @@ -229,11 +229,19 @@ class PipelineTaskList extends Component { } }; - handleRetry = async (taskId: number) => { + handleRetry = async (taskId: number, stageName?: string) => { try { - const res = await fetch(`/api/pipeline/tasks/${taskId}/retry`, { method: 'POST' }); + const body: any = {}; + if (stageName) { + body.stage_name = stageName; + } + const res = await fetch(`/api/pipeline/tasks/${taskId}/retry`, { + method: 'POST', + headers: stageName ? { 'Content-Type': 'application/json' } : undefined, + body: stageName ? JSON.stringify(body) : undefined, + }); if (res.ok) { - message.success('任务已重新排队'); + message.success(stageName ? `从 ${this.getStageLabel(stageName)} 阶段重试` : '任务已重新排队'); this.loadData(); } else { message.error('重试失败'); @@ -291,6 +299,7 @@ class PipelineTaskList extends Component { 'extract_cover': '提取封面', 'cloud_upload': '云盘上传', 'custom_command': '自定义命令', + 'burn_subtitles': '烧录字幕', }; return labels[stageName] || stageName; }; @@ -450,6 +459,9 @@ class PipelineTaskList extends Component { const results = task.stage_results || []; if (results.length === 0) return null; + // 是否允许阶段级重试（Failed/Cancelled/Completed 状态） + const canRetryStage = task.can_retry && (task.status === 'failed' || task.status === 'cancelled' || task.status === 'completed'); + return ( {results.map((result, index) => ( @@ -461,6 +473,19 @@ class PipelineTaskList extends Component { {result.status === 'failed' && ( 失败 )} + {canRetryStage && (result.status === 'completed' || result.status === 'failed') && ( + + )} } key={index} @@ -568,7 +593,7 @@ class PipelineTaskList extends Component { 取消任务 )} - {(task.status === 'failed' || task.status === 'cancelled') && task.can_retry && ( + {(task.status === 'failed' || task.status === 'cancelled' || task.status === 'completed') && task.can_retry && ( From d1f0703e821dd5aaecd6d21d42e031917197595e Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:59:35 +0800 Subject: [PATCH 05/52] =?UTF-8?q?fix(pipeline,openlist):=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E7=AE=A1=E9=81=93=E9=87=8D=E8=AF=95=E3=80=81=E5=B4=A9?= =?UTF-8?q?=E6=BA=83=E6=81=A2=E5=A4=8D=E5=92=8C=20OpenList=20=E7=94=9F?= =?UTF-8?q?=E5=91=BD=E5=91=A8=E6=9C=9F=E5=A4=9A=E4=B8=AA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline 修复： - 重试阶段过滤 Deletable/已上传文件，避免引用已删除路径导致重试失败 - 新增 onStageComplete 回调，每个阶段完成后持久化 CurrentFiles， 修复进程崩溃后恢复时 CurrentFiles 停留在初始文件的 bug - RetryTask 接受 completed 状态任务，与前端按钮和 RetryTaskFromStage 一致 - custom_command 区分中间产物和 delete_all 最终成品，不再跳过后者的命令执行 - 云上传路径碰撞处理使用 path.* 替代 filepath.*，修复 Windows 上反斜杠问题 OpenList 修复： - Start 失败时清除全局指针，避免永久阻止后续重试 - watchProcess 异常退出时关闭日志文件句柄，修复 fd 泄漏 - 新增 openlistLifecycleMu 串行化 enable/disable 回调，防止快速切换导致服务泄漏 - Start 完成后复核配置，如果在 waitForReady 期间已关闭则立即清理 - 云上传保持启用时检测 port/data_path 变化并自动重建 Manager 其他： - 重试接口拒绝格式错误的 JSON 请求体（仅忽略 io.EOF），防止误触发全量重试 --- src/cmd/bililive/bililive.go | 60 +++++++++++++++++++++++++++ src/pipeline/executor.go | 9 +++- src/pipeline/executor_test.go | 14 +++---- src/pipeline/manager.go | 28 ++++++++++++- src/pipeline/stages/custom_command.go | 17 ++++++-- src/pipeline/stages/extract_cover.go | 7 ++-- src/pkg/openlist/manager.go | 10 ++++- src/servers/pipeline_handler.go | 7 +++- 8 files changed, 132 insertions(+), 20 deletions(-) diff --git a/src/cmd/bililive/bililive.go b/src/cmd/bililive/bililive.go index a5e616d..f71dd59 100644 --- a/src/cmd/bililive/bililive.go +++ b/src/cmd/bililive/bililive.go @@ -339,6 +339,8 @@ func main() { // 如果启用了云上传功能，初始化 OpenList 管理器 var openlistManager *openlist.Manager + // 串行化 OpenList 生命周期操作，防止快速 enable/disable 导致服务泄漏 + var openlistLifecycleMu sync.Mutex if config.OnRecordFinished.CloudUpload.Enable { // 获取 OpenList 数据目录 openlistDataPath := config.OpenList.DataPath @@ -357,6 +359,9 @@ func main() { bilisentryPkg.Go(func() { if err := openlistManager.Start(rootCtx); err != nil { logger.WithError(err).Error("启动 OpenList 失败，云上传功能将不可用") + // 启动失败时清除全局指针，避免 GetGlobalManager() != nil 永久阻止后续重试 + servers.SetOpenListManager(nil) + openlistManager = nil } }) @@ -376,6 +381,8 @@ func main() { // 关闭云上传：停止 OpenList 进程 if wasEnabled && !isEnabled { + openlistLifecycleMu.Lock() + defer openlistLifecycleMu.Unlock() if mgr := openlist.GetGlobalManager(); mgr != nil { logger.Info("检测到云上传已关闭，正在停止 OpenList...") mgr.Stop() @@ -387,10 +394,49 @@ func main() { // 开启云上传：启动 OpenList 进程 if wasEnabled || !isEnabled { + // 云上传保持启用：检查是否需要重建 Manager（端口或数据目录变更） + if wasEnabled && isEnabled { + if mgr := openlist.GetGlobalManager(); mgr != nil { + // 解析新的配置值 + newPort := newCfg.OpenList.Port + if newPort == 0 { + newPort = 5244 + } + newDataPath := newCfg.OpenList.DataPath + if newDataPath == "" { + newDataPath = filepath.Join(newCfg.AppDataPath, "openlist") + } + // 与 Manager 当前值比较 + if mgr.GetPort() != newPort || mgr.GetDataPath() != newDataPath { + logger.Infof("检测到 OpenList 配置变更（端口: %d→%d, 数据目录: %s→%s），正在重建 Manager...", + mgr.GetPort(), newPort, mgr.GetDataPath(), newDataPath) + openlistLifecycleMu.Lock() + mgr.Stop() + servers.SetOpenListManager(nil) + // 创建新 Manager 并启动 + newMgr := openlist.NewManager(newDataPath, newPort) + servers.SetOpenListManager(newMgr) + openlistManager = newMgr + openlistLifecycleMu.Unlock() + bilisentryPkg.Go(func() { + if err := newMgr.Start(rootCtx); err != nil { + logger.WithError(err).Error("重建 OpenList 失败") + openlistLifecycleMu.Lock() + servers.SetOpenListManager(nil) + openlistManager = nil + openlistLifecycleMu.Unlock() + } + }) + } + } + } return } + + openlistLifecycleMu.Lock() // 已有 Manager 则跳过 if openlist.GetGlobalManager() != nil { + openlistLifecycleMu.Unlock() return } @@ -408,11 +454,25 @@ func main() { mgr := openlist.NewManager(openlistDataPath, openlistPort) // 先设置全局 Manager，让 API 端能立即获取到状态 servers.SetOpenListManager(mgr) + openlistLifecycleMu.Unlock() // 在后台启动 OpenList 进程（需要 rootCtx 生命周期） bilisentryPkg.Go(func() { if err := mgr.Start(rootCtx); err != nil { logger.WithError(err).Error("动态启动 OpenList 失败，云上传功能将不可用") + // 启动失败时清除全局指针，允许后续配置变更时重试 + openlistLifecycleMu.Lock() + servers.SetOpenListManager(nil) + openlistLifecycleMu.Unlock() + return + } + // Start 完成后复核配置：如果在 waitForReady 期间配置已关闭，立即清理 + openlistLifecycleMu.Lock() + defer openlistLifecycleMu.Unlock() + if cfg := configs.GetCurrentConfig(); cfg == nil || !cfg.OnRecordFinished.CloudUpload.Enable { + logger.Info("OpenList 启动完成但云上传已关闭，立即停止") + mgr.Stop() + servers.SetOpenListManager(nil) } }) }) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 19f0e4a..fc17cfe 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -49,12 +49,14 @@ func (e *Executor) getFactory(name string) (StageFactory, bool) { // Execute 执行管道 // startStage: 从第几个启用的阶段开始执行（0=全部执行，用于重试指定阶段） +// onStageComplete: 每个阶段成功完成后的回调，用于持久化进度（崩溃恢复的关键） func (e *Executor) Execute( ctx *PipelineContext, config *PipelineConfig, initialFiles []FileInfo, startStage int, onProgress func(stageIndex int, stageName string, status StageStatus), + onStageComplete func(stageIndex int, outputFiles []FileInfo), ) ([]StageResult, error) { if config == nil || len(config.Stages) == 0 { e.logger.Debug("pipeline config is empty, skipping") @@ -145,6 +147,11 @@ func (e *Executor) Execute( onProgress(stageIndex, stageCfg.Name, StageStatusCompleted) } + // 通知外部持久化阶段完成状态（用于崩溃恢复） + if onStageComplete != nil { + onStageComplete(stageIndex, output) + } + // 更新文件列表给下一阶段 files = output stageIndex++ @@ -468,7 +475,7 @@ func (e *Executor) ExecuteAsync( bilisentry.GoWithContext(ctx.Ctx, func(goCtx context.Context) { // 更新上下文 ctx.Ctx = goCtx - results, err := e.Execute(ctx, config, initialFiles, 0, onProgress) + results, err := e.Execute(ctx, config, initialFiles, 0, onProgress, nil) if onComplete != nil { onComplete(results, err) } diff --git a/src/pipeline/executor_test.go b/src/pipeline/executor_test.go index 97f6de3..1889eab 100644 --- a/src/pipeline/executor_test.go +++ b/src/pipeline/executor_test.go @@ -86,7 +86,7 @@ func TestExecute_AllStagesSucceed_AllDeletableFilesRemoved(t *testing.T) { results, err := executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, 0, nil) + }, 0, nil, nil) // 验证：无错误 assert.NoError(t, err) @@ -142,7 +142,7 @@ func TestExecute_StageFailed_AllFilesPreserved(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, 0, nil) + }, 0, nil, nil) // 验证：返回错误 assert.Error(t, err) @@ -199,7 +199,7 @@ func TestExecute_Cancelled_AllFilesPreserved(t *testing.T) { _, _ = executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, 0, nil) + }, 0, nil, nil) // 验证：stage2 返回成功，但下一轮循环检测到 context 已取消 // 注意：stage2 本身成功了，所以 deleteMarkedFiles 会被调用 @@ -248,7 +248,7 @@ func TestExecute_PassthroughPreservesDeletable(t *testing.T) { results, err := executor.Execute(ctx, config, []FileInfo{ {Path: file1, Type: FileTypeVideo}, {Path: file2, Type: FileTypeOther, Deletable: true}, - }, 0, nil) + }, 0, nil, nil) assert.NoError(t, err) assert.Len(t, results, 1) @@ -304,7 +304,7 @@ func TestExecute_UploadCancelledDuringUpload_FilePreserved(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: localFile, Type: FileTypeVideo}, - }, 0, nil) + }, 0, nil, nil) // 验证：返回错误 assert.Error(t, err) @@ -363,7 +363,7 @@ func TestExecute_UploadFailsMiddleOfFileList_NoneDeleted(t *testing.T) { {Path: file1, Type: FileTypeVideo}, {Path: file2, Type: FileTypeVideo}, {Path: file3, Type: FileTypeVideo}, - }, 0, nil) + }, 0, nil, nil) assert.NoError(t, err) @@ -423,7 +423,7 @@ func TestExecute_StaleDeletableFromDB_ClearedOnFailure(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: file1, Type: FileTypeVideo, Deletable: true}, // 旧标记 {Path: file2, Type: FileTypeVideo, Deletable: true}, // 旧标记 - }, 0, nil) + }, 0, nil, nil) assert.NoError(t, err) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index ff2c3d6..f5d66b4 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -226,6 +226,13 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { } m.broadcastTaskUpdate(task) }, + // 阶段完成后持久化 CurrentFiles，确保崩溃恢复时能拿到正确的中间文件 + func(stageIndex int, outputFiles []FileInfo) { + task.CurrentFiles = outputFiles + if err := m.store.UpdateTask(ctx, task); err != nil { + logrus.WithError(err).Warn("failed to persist stage output files") + } + }, ) // 合并：保留之前的结果 + 新执行的结果 @@ -347,8 +354,9 @@ func (m *Manager) RetryTask(taskID int64) error { return fmt.Errorf("task cannot be retried") } - if task.Status != PipelineStatusFailed && task.Status != PipelineStatusCancelled { - return fmt.Errorf("only failed or cancelled tasks can be retried") + // 允许 Failed/Cancelled/Completed 状态 + if task.Status != PipelineStatusFailed && task.Status != PipelineStatusCancelled && task.Status != PipelineStatusCompleted { + return fmt.Errorf("only failed, cancelled or completed tasks can be retried") } // 重置任务状态 @@ -436,6 +444,22 @@ func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error { inputFiles = task.InitialFiles } + // 过滤掉已标记删除/已上传的中间文件（它们在管道完成后已被 deleteMarkedFiles 从磁盘移除） + // 只保留目标阶段实际需要的文件，确保后续 os.Stat 校验和重试执行不会引用已删除的路径 + var activeFiles []FileInfo + for _, f := range inputFiles { + if f.Deletable { + continue + } + if f.Metadata != nil { + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + continue + } + } + activeFiles = append(activeFiles, f) + } + inputFiles = activeFiles + // 校验输入文件不为空 if len(inputFiles) == 0 { return fmt.Errorf("no input files available for stage %s, cannot retry", stageName) diff --git a/src/pipeline/stages/custom_command.go b/src/pipeline/stages/custom_command.go index 50b7186..cdcb316 100644 --- a/src/pipeline/stages/custom_command.go +++ b/src/pipeline/stages/custom_command.go @@ -51,9 +51,10 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe var output []pipeline.FileInfo for _, file := range input { - // 跳过已标记待删除的文件（如 ConvertMp4/BurnSubtitles 阶段标记的中间产物）， - // 避免对本应删除的文件重复执行命令，但仍将其透传到 output。 - if file.Deletable { + // 跳过中间产物（如 ConvertMp4 标记的原始 FLV），避免重复执行命令。 + // 但 delete_all_after_upload 模式会将所有文件（含最终成品）标记为 Deletable， + // 此类文件仍需执行自定义命令，不能跳过。 + if file.Deletable && !isDeleteAllMarked(file) { output = append(output, file) continue } @@ -84,6 +85,16 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe return output, nil } +// isDeleteAllMarked 检查文件是否被 delete_all_after_upload 模式标记 +// 此类文件虽为 Deletable，但属于最终成品，仍需执行自定义命令 +func isDeleteAllMarked(file pipeline.FileInfo) bool { + if file.Metadata == nil { + return false + } + da, ok := file.Metadata["delete_all"].(bool) + return ok && da +} + // renderCommand 渲染命令模板 func (s *CustomCommandStage) renderCommand(ctx *pipeline.PipelineContext, file pipeline.FileInfo) (string, error) { // 构建模板数据 diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index f8224b3..af42e4a 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -212,10 +212,11 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 检测远程路径冲突（多分段视频或视频+封面使用相同模板时可能产生同路径） // 冲突时自动追加原始文件名以区分 if usedPaths[targetPath] { - dir := filepath.Dir(targetPath) + // 使用 path.* 而非 filepath.*，因为 targetPath 是远程路径，始终用正斜杠 + dir := path.Dir(targetPath) ext := filepath.Ext(targetPath) - base := strings.TrimSuffix(filepath.Base(targetPath), ext) - targetPath = filepath.Join(dir, base+"_"+filepath.Base(file.Path)) + base := strings.TrimSuffix(path.Base(targetPath), ext) + targetPath = path.Join(dir, base+"_"+filepath.Base(file.Path)) ctx.Logger.Infof("检测到远程路径冲突，自动追加文件名: %s", targetPath) } usedPaths[targetPath] = true diff --git a/src/pkg/openlist/manager.go b/src/pkg/openlist/manager.go index d2a437d..28f9993 100644 --- a/src/pkg/openlist/manager.go +++ b/src/pkg/openlist/manager.go @@ -266,10 +266,16 @@ func (m *Manager) watchProcess() { select { case <-m.stopCh: - // 正常停止 + // 正常停止（stopInternal 已负责关闭 m.logFile） default: - // 异常退出 + // 异常退出：关闭日志文件句柄，避免 fd 泄漏 if wasRunning { + m.mu.Lock() + if m.logFile != nil { + m.logFile.Close() + m.logFile = nil + } + m.mu.Unlock() logrus.WithError(err).Warn("OpenList 进程异常退出") } } diff --git a/src/servers/pipeline_handler.go b/src/servers/pipeline_handler.go index ea0e584..f01676c 100644 --- a/src/servers/pipeline_handler.go +++ b/src/servers/pipeline_handler.go @@ -2,6 +2,7 @@ package servers import ( "encoding/json" + "io" "net/http" "os" "path/filepath" @@ -181,8 +182,10 @@ func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc { StageName string `json:"stage_name"` } if r.Body != nil { - // 尝试解析 body，失败不影响（向后兼容无 body 的情况） - json.NewDecoder(r.Body).Decode(&reqBody) + if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil && err != io.EOF { + http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest) + return + } } if reqBody.StageName != "" { From ea31f8b3d30be26cd66505adc43d3277a1c19a17 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:10:37 +0800 Subject: [PATCH 06/52] =?UTF-8?q?fix(config):=20=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E6=A8=A1=E6=9D=BF=E9=BB=98=E8=AE=A4=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E5=8E=9F=E5=A7=8B=E6=96=87=E4=BB=B6=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将云上传路径模板默认值从「简洁归档」改为「原始文件名」预设， 新用户启用云上传时直接保留原始文件名，减少配置门槛。 同步更新 config.yml 避免全新环境启动时被旧值覆盖。 --- config.yml | 55 ++++++++++++++++++++++++++++++++++++++++++- src/configs/config.go | 2 +- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/config.yml b/config.yml index 0cd83aa..7206f46 100644 --- a/config.yml +++ b/config.yml @@ -30,6 +30,8 @@ video_split_strategies: max_file_size: "0" on_record_finished: convert_to_mp4: false + # MP4 转换成功后标记原始 FLV 为待删除，全部处理阶段完成后才真正删除 + # 默认 false（保留原始 FLV 文件） delete_flv_after_convert: false # 当 custom_commandline 的值 不为空时，convert_to_mp4 的值会被无视， # 而是在录制结束后直接执行 custom_commandline 中的命令。 @@ -43,10 +45,55 @@ on_record_finished: cloud_upload: enable: false storage_name: "" - upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }} + upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} + # 选「处理完再上传」时，上传成功后仅删除已上传的文件（如最终视频），不影响其他中间文件 + # 选「先上传再处理」时此开关无效 delete_after_upload: false + # 选「处理完再上传」时，上传成功后删除所有本地文件（含中间产物） + # 选「先上传再处理」时此开关无效 + delete_all_after_upload: false upload_timing: after_process + # 是否将 ASS 弹幕字幕硬编码（烧录）到视频中 + # 开启后会使用 FFmpeg 重编码视频，将字幕叠加到画面上 + # 需要同时开启弹幕录制（danmaku_enable）才有 ASS 文件可烧录 + burn_subtitles: false + # 烧录字幕时使用的视频编码器 + # 默认 libx264（H.264），兼容性最好 + # 可选 libx265（H.265，压缩率更高但编码更慢） + burn_subtitles_codec: libx264 + # 烧录字幕时的 CRF（恒定速率因子）质量值 + # 范围 0-51，数值越小画质越好，文件越大 + # 默认 18（视觉无损），推荐范围 15-23 + burn_subtitles_crf: "18" + # 烧录字幕时的编码预设 + # 影响编码速度和压缩率的平衡 + # 可选: ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow + # 默认 medium，越慢画质越好但耗时越长 + burn_subtitles_preset: medium + # 烧录成功后标记 ASS 文件为待删除，全部处理阶段完成后才真正删除 + # 默认 false（保留 ASS 文件） + burn_delete_ass: false + # 烧录成功后标记源视频文件为待删除，全部处理阶段完成后才真正删除 + # 默认 false（保留源文件，同时存在源文件和烧录后的 MKV） + # 开启后仅保留烧录完成的 MKV 文件 + burn_delete_source: false timeout_in_us: 60000000 +danmaku_enable: false +danmaku: + font_size: 36 + font_name: Microsoft YaHei + scroll_area: full + scroll_time: 10 + resolution: 1920x1080 + outline: 1 + opacity: 128 + record_gift: true + record_douyu_gift: true + record_douyin_gift: true + record_guard: true + record_super_chat: true + guard_position: bottom-left + sc_position: bottom-left live_rooms: # quality参数目前仅B站启用，默认为0 # (B站)0代表原画PRO(HEVC)优先, 其他数值为原画(AVC) @@ -106,6 +153,10 @@ notify: icon: "" # 通知级别（可选）: active/timeSensitive/passive/critical level: "" + wxpusher: + enable: false + appToken: "" + uids: [] app_data_path: .appdata read_only_tool_folder: "" tool_root_folder: "" @@ -126,6 +177,8 @@ proxy: openlist: port: 5244 data_path: "" + username: "" + password: "" update: auto_check: true check_interval_hours: 6 diff --git a/src/configs/config.go b/src/configs/config.go index 85a2d92..4a46402 100644 --- a/src/configs/config.go +++ b/src/configs/config.go @@ -1012,7 +1012,7 @@ var defaultConfig = Config{ CloudUpload: CloudUpload{ Enable: false, StorageName: "", - UploadPathTmpl: "/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date \"2006-01-02\" }}.{{ .Ext }}", + UploadPathTmpl: "/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }}", DeleteAfterUpload: false, }, UploadTiming: UploadTimingAfterProcess, From 589586837e5a2cb9eef47407d7f2170e8057f393 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:25:39 +0800 Subject: [PATCH 07/52] =?UTF-8?q?chore(config):=20=E7=A7=BB=E9=99=A4=20con?= =?UTF-8?q?fig.yml=20=E4=B8=AD=E5=BC=B9=E5=B9=95=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E5=86=97=E4=BD=99=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.yml | 30 --------------------- tests/e2e/fixtures/test-config.template.yml | 2 +- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/config.yml b/config.yml index 7206f46..6eaad1e 100644 --- a/config.yml +++ b/config.yml @@ -30,15 +30,7 @@ video_split_strategies: max_file_size: "0" on_record_finished: convert_to_mp4: false - # MP4 转换成功后标记原始 FLV 为待删除，全部处理阶段完成后才真正删除 - # 默认 false（保留原始 FLV 文件） delete_flv_after_convert: false - # 当 custom_commandline 的值 不为空时，convert_to_mp4 的值会被无视， - # 而是在录制结束后直接执行 custom_commandline 中的命令。 - # 在 custom_commandline 执行结束后，程序还会继续查看 delete_flv_after_convert 的值， - # 来判断是否需要删除原始 flv 文件。 - # 以下是一个在录制结束后将 flv 视频转换为同名 mp4 视频的示例： - # custom_commandline: '{{ .Ffmpeg }} -hide_banner -i "{{ .FileName }}" -c copy "{{ .FileName | trimSuffix (.FileName | ext)}}.mp4"' custom_commandline: "" fix_flv_at_first: true save_cover: false @@ -46,36 +38,14 @@ on_record_finished: enable: false storage_name: "" upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} - # 选「处理完再上传」时，上传成功后仅删除已上传的文件（如最终视频），不影响其他中间文件 - # 选「先上传再处理」时此开关无效 delete_after_upload: false - # 选「处理完再上传」时，上传成功后删除所有本地文件（含中间产物） - # 选「先上传再处理」时此开关无效 delete_all_after_upload: false upload_timing: after_process - # 是否将 ASS 弹幕字幕硬编码（烧录）到视频中 - # 开启后会使用 FFmpeg 重编码视频，将字幕叠加到画面上 - # 需要同时开启弹幕录制（danmaku_enable）才有 ASS 文件可烧录 burn_subtitles: false - # 烧录字幕时使用的视频编码器 - # 默认 libx264（H.264），兼容性最好 - # 可选 libx265（H.265，压缩率更高但编码更慢） burn_subtitles_codec: libx264 - # 烧录字幕时的 CRF（恒定速率因子）质量值 - # 范围 0-51，数值越小画质越好，文件越大 - # 默认 18（视觉无损），推荐范围 15-23 burn_subtitles_crf: "18" - # 烧录字幕时的编码预设 - # 影响编码速度和压缩率的平衡 - # 可选: ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow - # 默认 medium，越慢画质越好但耗时越长 burn_subtitles_preset: medium - # 烧录成功后标记 ASS 文件为待删除，全部处理阶段完成后才真正删除 - # 默认 false（保留 ASS 文件） burn_delete_ass: false - # 烧录成功后标记源视频文件为待删除，全部处理阶段完成后才真正删除 - # 默认 false（保留源文件，同时存在源文件和烧录后的 MKV） - # 开启后仅保留烧录完成的 MKV 文件 burn_delete_source: false timeout_in_us: 60000000 danmaku_enable: false diff --git a/tests/e2e/fixtures/test-config.template.yml b/tests/e2e/fixtures/test-config.template.yml index 0c7b8a5..54f6268 100644 --- a/tests/e2e/fixtures/test-config.template.yml +++ b/tests/e2e/fixtures/test-config.template.yml @@ -43,7 +43,7 @@ on_record_finished: cloud_upload: enable: false storage_name: "" - upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }} + upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} delete_after_upload: false upload_timing: after_process timeout_in_us: 60000000 From ae599e1e0ccb82d4fa83d379a5e554d5a764d85e Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:45:26 +0800 Subject: [PATCH 08/52] =?UTF-8?q?chore(config):=20=E7=A7=BB=E9=99=A4=20con?= =?UTF-8?q?fig=5Fcomments.go=20=E4=B8=AD=E5=BC=B9=E5=B9=95=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E5=86=97=E4=BD=99=E6=B3=A8=E9=87=8A=E7=94=9F=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 5895868 仅从 config.yml 删除了注释文本，但 DecorateConfigNode() 在程序启动时仍会重新注入这些注释。本次从源头移除以下 setFieldComment 调用： - on_record_finished: custom_commandline, delete_flv_after_convert - burn_subtitles 相关: codec, crf, preset, delete_ass, delete_source - cloud_upload: delete_after_upload, delete_all_after_upload Co-Authored-By: Claude Opus 4.8 --- src/configs/config_comments.go | 55 ---------------------------------- 1 file changed, 55 deletions(-) diff --git a/src/configs/config_comments.go b/src/configs/config_comments.go index b8c0a24..830eac9 100644 --- a/src/configs/config_comments.go +++ b/src/configs/config_comments.go @@ -32,61 +32,6 @@ func DecorateConfigNode(node *yaml.Node) { # 负数为非法值，程序会输出 log 提醒，并无视所设定的数值`, "") } - finishNode := findNode(root, "on_record_finished") - if finishNode != nil { - setFieldComment(finishNode, "custom_commandline", - `# 当 custom_commandline 的值 不为空时，convert_to_mp4 的值会被无视， -# 而是在录制结束后直接执行 custom_commandline 中的命令。 -# 在 custom_commandline 执行结束后，程序还会继续查看 delete_flv_after_convert 的值， -# 来判断是否需要删除原始 flv 文件。 -# 以下是一个在录制结束后将 flv 视频转换为同名 mp4 视频的示例： -# custom_commandline: '{{ .Ffmpeg }} -hide_banner -i "{{ .FileName }}" -c copy "{{ .FileName | trimSuffix (.FileName | ext)}}.mp4"'`, "") - - setFieldComment(finishNode, "delete_flv_after_convert", - `# MP4 转换成功后标记原始 FLV 为待删除，全部处理阶段完成后才真正删除 -# 默认 false（保留原始 FLV 文件）`, "") - - setFieldComment(finishNode, "burn_subtitles", - `# 是否将 ASS 弹幕字幕硬编码（烧录）到视频中 -# 开启后会使用 FFmpeg 重编码视频，将字幕叠加到画面上 -# 需要同时开启弹幕录制（danmaku_enable）才有 ASS 文件可烧录`, "") - - setFieldComment(finishNode, "burn_subtitles_codec", - `# 烧录字幕时使用的视频编码器 -# 默认 libx264（H.264），兼容性最好 -# 可选 libx265（H.265，压缩率更高但编码更慢）`, "") - - setFieldComment(finishNode, "burn_subtitles_crf", - `# 烧录字幕时的 CRF（恒定速率因子）质量值 -# 范围 0-51，数值越小画质越好，文件越大 -# 默认 18（视觉无损），推荐范围 15-23`, "") - - setFieldComment(finishNode, "burn_subtitles_preset", - `# 烧录字幕时的编码预设 -# 影响编码速度和压缩率的平衡 -# 可选: ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow -# 默认 medium，越慢画质越好但耗时越长`, "") - - setFieldComment(finishNode, "burn_delete_ass", - `# 烧录成功后标记 ASS 文件为待删除，全部处理阶段完成后才真正删除 -# 默认 false（保留 ASS 文件）`, "") - - setFieldComment(finishNode, "burn_delete_source", - `# 烧录成功后标记源视频文件为待删除，全部处理阶段完成后才真正删除 -# 默认 false（保留源文件，同时存在源文件和烧录后的 MKV） -# 开启后仅保留烧录完成的 MKV 文件`, "") - - cloudNode := findNode(finishNode, "cloud_upload") - if cloudNode != nil { - setFieldComment(cloudNode, "delete_after_upload", - `# 选「处理完再上传」时，上传成功后仅删除已上传的文件（如最终视频），不影响其他中间文件 -# 选「先上传再处理」时此开关无效`, "") - setFieldComment(cloudNode, "delete_all_after_upload", - `# 选「处理完再上传」时，上传成功后删除所有本地文件（含中间产物） -# 选「先上传再处理」时此开关无效`, "") - } - } - setFieldHeadComment(root, "notify", "# 通知服务配置") notifyNode := findNode(root, "notify") if notifyNode != nil { From 8cedef8c7ab9b61a984b5f83b8fa8451716d718c Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:01:37 +0800 Subject: [PATCH 09/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D=20del?= =?UTF-8?q?ete=5Fafter=20=E6=A8=A1=E5=BC=8F=E4=B8=8B=E5=AD=A4=E7=AB=8B=20.?= =?UTF-8?q?ass=20=E6=96=87=E4=BB=B6=E6=9C=AA=E8=A2=AB=E6=B8=85=E7=90=86?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deleteMarkedFiles 中仅在 delete_all 模式下运行 cleanupOrphanedAssFiles， 导致 delete_after 模式上传删除视频后 .ass 文件残留在磁盘。 - 新增 deleteAfter 标志检测，使孤立 .ass 清理在两种删除模式下均生效 - cleanupOrphanedAssFiles 改用视频基名匹配替代 allAssFiles 过滤， 解决 .ass 从未传入 Pipeline 导致过滤条件失效的问题 --- src/pipeline/executor.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index fc17cfe..10edf4c 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -182,14 +182,18 @@ func (e *Executor) Execute( // deleteMarkedFiles 删除标记为 Deletable 或 Metadata["uploaded"] 的文件，返回保留的文件列表 func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { var kept []FileInfo - deleteAll := false // 标记是否为 deleteAll 模式 + deleteAll := false // 标记是否为 deleteAll 模式 + deleteAfter := false // 标记是否为 deleteAfter 模式（仅删除已上传文件） for _, f := range files { - // 检查是否为 deleteAll 模式（CloudUploadStage 标记） + // 检查删除模式（CloudUploadStage 标记） if f.Metadata != nil { if da, ok := f.Metadata["delete_all"].(bool); ok && da { deleteAll = true } + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + deleteAfter = true + } } // 判断是否应删除：Deletable 标记 或 CloudUploadStage 标记的已上传文件 @@ -214,9 +218,9 @@ func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { } } - // 仅在 deleteAll 模式下清理孤立 .ass 文件 - // deleteAfter 模式只删除已上传文件，不应连带清理弹幕 - if deleteAll { + // 清理无关联视频文件的孤立 .ass 文件 + // deleteAll 模式下删除所有文件（含 .ass），deleteAfter 模式下删除已上传视频后也应清理残留 .ass + if deleteAll || deleteAfter { kept = e.cleanupOrphanedAssFiles(files, kept) } @@ -242,21 +246,14 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File // 收集 Pipeline 中已知的 .ass 文件路径 knownAssFiles := map[string]bool{} - // 同时收集 allFiles 中的 .ass 文件路径（用于限定扫描范围） - allAssFiles := map[string]bool{} for _, f := range keptFiles { if strings.ToLower(filepath.Ext(f.Path)) == ".ass" { knownAssFiles[f.Path] = true } } - for _, f := range allFiles { - if strings.ToLower(filepath.Ext(f.Path)) == ".ass" { - allAssFiles[f.Path] = true - } - } // 扫描目录中的 .ass 文件 - // 仅处理本任务关联的 .ass 文件，避免误删其他任务或用户放置的字幕 + // 通过视频基名匹配判断 .ass 是否属于本任务，避免误删其他任务或用户放置的字幕 for dir := range dirs { entries, err := os.ReadDir(dir) if err != nil { @@ -273,13 +270,14 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File if knownAssFiles[assPath] { continue // 已在 Pipeline 中处理 } - if !allAssFiles[assPath] { - continue // 非本任务关联的 .ass 文件，跳过 - } - // 检查磁盘上是否有同名视频文件 + // 通过基名匹配判断是否属于本任务的视频文件 + // （.ass 文件可能未被传入 Pipeline，不能依赖文件列表过滤） base := strings.TrimSuffix(entry.Name(), ".ass") key := filepath.Join(dir, base) - // 检查磁盘上实际存在的视频文件（含 .ts 格式） + if !videoBaseNames[key] { + continue // 基名不匹配任何本任务视频，跳过 + } + // 检查磁盘上是否有同名视频文件 hasVideo := false for _, ext := range []string{".flv", ".mp4", ".mkv", ".ts"} { if _, err := os.Stat(key + ext); err == nil { From 592b8966a1218aedbb1521b6af1b00d8c2f0b661 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:01:57 +0800 Subject: [PATCH 10/52] =?UTF-8?q?chore(config):=20=E7=83=A7=E5=BD=95?= =?UTF-8?q?=E5=AD=97=E5=B9=95=E6=97=B6=E5=88=A0=E9=99=A4=E6=BA=90=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E6=96=87=E4=BB=B6=E9=81=BF=E5=85=8D=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit burn_delete_source 改为 true，烧录后源 .flv 标记为中间产物跳过上传， 仅上传最终成品 .mkv，节省带宽和云存储空间。 --- config.yml | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/config.yml b/config.yml index 6eaad1e..975666e 100644 --- a/config.yml +++ b/config.yml @@ -21,7 +21,7 @@ feature: out_put_tmpl: "" video_split_strategies: on_room_name_changed: false - max_duration: 0s + max_duration: 1m20s # 仅在使用 ffmpeg 或 bililive-recorder 下载器时生效 # 支持可读格式，如: 500MB, 1GB, 1.5GB, 1024KB # 也支持纯数字（视为字节），如: 1073741824 @@ -30,25 +30,25 @@ video_split_strategies: max_file_size: "0" on_record_finished: convert_to_mp4: false - delete_flv_after_convert: false + delete_flv_after_convert: true custom_commandline: "" fix_flv_at_first: true save_cover: false cloud_upload: - enable: false - storage_name: "" - upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} - delete_after_upload: false + enable: true + storage_name: baidu + upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}/{{ .FileName }} + delete_after_upload: true delete_all_after_upload: false upload_timing: after_process - burn_subtitles: false + burn_subtitles: true burn_subtitles_codec: libx264 burn_subtitles_crf: "18" burn_subtitles_preset: medium burn_delete_ass: false - burn_delete_source: false + burn_delete_source: true timeout_in_us: 60000000 -danmaku_enable: false +danmaku_enable: true danmaku: font_size: 36 font_name: Microsoft YaHei @@ -69,12 +69,9 @@ live_rooms: # (B站)0代表原画PRO(HEVC)优先, 其他数值为原画(AVC) # 原画PRO会保存为.ts文件, 原画为.flv # HEVC相比AVC体积更小, 减少35%体积, 画质相当, 但是B站转码有时候会崩 - - url: https://www.lang.live/room/5664344 + - url: https://live.bilibili.com/1601605?live_from=71002&visit_id=cp0crnd7mt40 is_listening: false scheme: "" - - url: https://live.bilibili.com/22603245 - is_listening: true - scheme: "" cookies: {} # 通知服务配置 notify: @@ -147,8 +144,9 @@ proxy: openlist: port: 5244 data_path: "" - username: "" - password: "" + username: admin + password: RcoNhw4s + token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwicHdkX3RzIjowLCJleHAiOjE3ODY1OTc4NjcsIm5iZiI6MTc4NjQyNTA2NywiaWF0IjoxNzg2NDI1MDY3fQ.pB4bk71C-gcsObup1acrfq2nxHX16SSWeGoSouzotWY update: auto_check: true check_interval_hours: 6 From a170b6c7c68d00d55367237766acca044fade106 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:12:36 +0800 Subject: [PATCH 11/52] =?UTF-8?q?fix(webapp):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=84=E7=90=86=E9=A2=84=E8=A7=88=E4=B8=8E?= =?UTF-8?q?=E5=AE=9E=E9=99=85=E4=B8=8A=E4=BC=A0=E8=A1=8C=E4=B8=BA=E4=B8=8D?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit burn_delete_source=false 时，BurnSubtitlesStage 输出的源视频和烧录视频 都会被 CloudUploadStage 上传，但预览只显示烧录视频。 - analyzeConfig 中 burn+!burnDelSource 时将源视频也标记为 uploaded - delAfter 删除逻辑同步处理被上传的源视频 --- .../config-info/CloudUploadSettings.tsx | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index e205d13..c472365 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -173,9 +173,16 @@ const analyzeConfig = (config: any) => { // after_process 模式：上传最终文件 if (isAfterProcess && upload) { - // 上传第一个视频和封面 + // 上传最终产物 if (burn) { files['video.mkv'].status = 'uploaded'; + // burn_delete_source=false 时，源视频也会被上传（未标记 Deletable） + if (!burnDelSource) { + const src = convert ? 'video.mp4' : 'video.flv'; + if (files[src] && files[src].status === 'kept') { + files[src].status = 'uploaded'; + } + } } else if (convert) { files['video.mp4'].status = 'uploaded'; } else { @@ -196,9 +203,20 @@ const analyzeConfig = (config: any) => { if (cover) files['cover.jpg'].status = 'uploaded_deleted'; } else if (delAfter) { // 只删除已上传的文件 - if (burn) files['video.mkv'].status = 'uploaded_deleted'; - else if (convert) files['video.mp4'].status = 'uploaded_deleted'; - else files['video.flv'].status = 'uploaded_deleted'; + if (burn) { + files['video.mkv'].status = 'uploaded_deleted'; + // burn_delete_source=false 时源视频也被上传，应一并删除 + if (!burnDelSource) { + const src = convert ? 'video.mp4' : 'video.flv'; + if (files[src] && files[src].status === 'uploaded') { + files[src].status = 'uploaded_deleted'; + } + } + } else if (convert) { + files['video.mp4'].status = 'uploaded_deleted'; + } else { + files['video.flv'].status = 'uploaded_deleted'; + } if (cover) files['cover.jpg'].status = 'uploaded_deleted'; } } From 28c7f0d7ada992bf1e12b285389b1791fd25e501 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:18:22 +0800 Subject: [PATCH 12/52] =?UTF-8?q?fix(webapp):=20=E5=85=A8=E9=9D=A2?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=96=87=E4=BB=B6=E5=A4=84=E7=90=86=E9=A2=84?= =?UTF-8?q?=E8=A7=88=E4=B8=8E=E5=AE=9E=E9=99=85=E8=A1=8C=E4=B8=BA=E4=B8=8D?= =?UTF-8?q?=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - burn+!burnDelSource 时，源视频在烧录阶段即标记为 uploaded - delAllAfter 改为通用逻辑：保存已上传状态 → 全部删除 → 恢复已上传标记 - delAfter 简化为通用遍历，不再逐文件硬编码 - 修复 immediate+burn+!burnDelSource 时源视频未显示为已上传的问题 --- .../config-info/CloudUploadSettings.tsx | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index c472365..b2e047a 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -160,6 +160,13 @@ const analyzeConfig = (config: any) => { } else { files[target].status = 'deleted'; } + } else { + // burn_delete_source=false：源视频保留在 BurnSubtitlesStage output 中 + // immediate 模式下已在 pipeline 开头被上传，after_process 模式下也会被上传 + const src = convert ? 'video.mp4' : 'video.flv'; + if (files[src] && files[src].status === 'kept') { + files[src].status = 'uploaded'; + } } if (burnDelAss) { files['video.ass'].status = 'deleted'; @@ -176,13 +183,7 @@ const analyzeConfig = (config: any) => { // 上传最终产物 if (burn) { files['video.mkv'].status = 'uploaded'; - // burn_delete_source=false 时，源视频也会被上传（未标记 Deletable） - if (!burnDelSource) { - const src = convert ? 'video.mp4' : 'video.flv'; - if (files[src] && files[src].status === 'kept') { - files[src].status = 'uploaded'; - } - } + // 源视频已在烧录阶段标记为 uploaded（burnDelSource=false 时） } else if (convert) { files['video.mp4'].status = 'uploaded'; } else { @@ -194,30 +195,21 @@ const analyzeConfig = (config: any) => { // 删除逻辑 if (delAllAfter) { - // 删除全部 + // 删除全部：先保存已上传文件的状态，再全部标记删除，最后恢复已上传的标记 + const uploadedStatus: Record = {}; + Object.entries(files).forEach(([name, f]) => { + if (f.status === 'uploaded') uploadedStatus[name] = f.status; + }); Object.values(files).forEach(f => { f.status = 'deleted'; }); - // 但上传的文件标记为 uploaded_deleted - if (burn) files['video.mkv'].status = 'uploaded_deleted'; - else if (convert) files['video.mp4'].status = 'uploaded_deleted'; - else files['video.flv'].status = 'uploaded_deleted'; - if (cover) files['cover.jpg'].status = 'uploaded_deleted'; + // 恢复已上传文件为 uploaded_deleted + Object.entries(uploadedStatus).forEach(([name]) => { + files[name].status = 'uploaded_deleted'; + }); } else if (delAfter) { // 只删除已上传的文件 - if (burn) { - files['video.mkv'].status = 'uploaded_deleted'; - // burn_delete_source=false 时源视频也被上传，应一并删除 - if (!burnDelSource) { - const src = convert ? 'video.mp4' : 'video.flv'; - if (files[src] && files[src].status === 'uploaded') { - files[src].status = 'uploaded_deleted'; - } - } - } else if (convert) { - files['video.mp4'].status = 'uploaded_deleted'; - } else { - files['video.flv'].status = 'uploaded_deleted'; - } - if (cover) files['cover.jpg'].status = 'uploaded_deleted'; + Object.entries(files).forEach(([, f]) => { + if (f.status === 'uploaded') f.status = 'uploaded_deleted'; + }); } } From 19108b594cb8091e74c46633b11a1e158e8f0b38 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:20:02 +0800 Subject: [PATCH 13/52] =?UTF-8?q?Revert=20"chore(config):=20=E7=83=A7?= =?UTF-8?q?=E5=BD=95=E5=AD=97=E5=B9=95=E6=97=B6=E5=88=A0=E9=99=A4=E6=BA=90?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E6=96=87=E4=BB=B6=E9=81=BF=E5=85=8D=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E4=B8=8A=E4=BC=A0"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 592b8966a1218aedbb1521b6af1b00d8c2f0b661. --- config.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/config.yml b/config.yml index 975666e..6eaad1e 100644 --- a/config.yml +++ b/config.yml @@ -21,7 +21,7 @@ feature: out_put_tmpl: "" video_split_strategies: on_room_name_changed: false - max_duration: 1m20s + max_duration: 0s # 仅在使用 ffmpeg 或 bililive-recorder 下载器时生效 # 支持可读格式，如: 500MB, 1GB, 1.5GB, 1024KB # 也支持纯数字（视为字节），如: 1073741824 @@ -30,25 +30,25 @@ video_split_strategies: max_file_size: "0" on_record_finished: convert_to_mp4: false - delete_flv_after_convert: true + delete_flv_after_convert: false custom_commandline: "" fix_flv_at_first: true save_cover: false cloud_upload: - enable: true - storage_name: baidu - upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}/{{ .FileName }} - delete_after_upload: true + enable: false + storage_name: "" + upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} + delete_after_upload: false delete_all_after_upload: false upload_timing: after_process - burn_subtitles: true + burn_subtitles: false burn_subtitles_codec: libx264 burn_subtitles_crf: "18" burn_subtitles_preset: medium burn_delete_ass: false - burn_delete_source: true + burn_delete_source: false timeout_in_us: 60000000 -danmaku_enable: true +danmaku_enable: false danmaku: font_size: 36 font_name: Microsoft YaHei @@ -69,9 +69,12 @@ live_rooms: # (B站)0代表原画PRO(HEVC)优先, 其他数值为原画(AVC) # 原画PRO会保存为.ts文件, 原画为.flv # HEVC相比AVC体积更小, 减少35%体积, 画质相当, 但是B站转码有时候会崩 - - url: https://live.bilibili.com/1601605?live_from=71002&visit_id=cp0crnd7mt40 + - url: https://www.lang.live/room/5664344 is_listening: false scheme: "" + - url: https://live.bilibili.com/22603245 + is_listening: true + scheme: "" cookies: {} # 通知服务配置 notify: @@ -144,9 +147,8 @@ proxy: openlist: port: 5244 data_path: "" - username: admin - password: RcoNhw4s - token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwicHdkX3RzIjowLCJleHAiOjE3ODY1OTc4NjcsIm5iZiI6MTc4NjQyNTA2NywiaWF0IjoxNzg2NDI1MDY3fQ.pB4bk71C-gcsObup1acrfq2nxHX16SSWeGoSouzotWY + username: "" + password: "" update: auto_check: true check_interval_hours: 6 From 58dd72fd1bcb1aa3382b3ce2c2c24e0c55bd7d53 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:40:36 +0800 Subject: [PATCH 14/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D=20del?= =?UTF-8?q?ete=5Fafter=20=E6=A8=A1=E5=BC=8F=E5=AD=A4=E7=AB=8B=20.ass=20?= =?UTF-8?q?=E6=9C=AA=E6=B8=85=E7=90=86=E5=8F=8A=E5=8F=96=E6=B6=88=E5=90=8E?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=8A=B6=E6=80=81=E4=B8=8D=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - deleteMarkedFiles: deleteAfter 模式下也运行 cleanupOrphanedAssFiles 修复视频上传删除后 .ass 文件残留在磁盘的问题 - cleanupOrphanedAssFiles: 改用视频基名匹配替代 allAssFiles 过滤 解决 .ass 从未传入 Pipeline 导致过滤条件失效的问题 - Execute: 全部阶段成功但 ctx 取消时返回 ctx.Err() 而非 nil 使任务标记为 Cancelled 而非 Completed，与磁盘状态（文件保留）一致 --- src/pipeline/executor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 10edf4c..0949aec 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -56,7 +56,7 @@ func (e *Executor) Execute( initialFiles []FileInfo, startStage int, onProgress func(stageIndex int, stageName string, status StageStatus), - onStageComplete func(stageIndex int, outputFiles []FileInfo), + onStageComplete func(stageIndex int, result StageResult), ) ([]StageResult, error) { if config == nil || len(config.Stages) == 0 { e.logger.Debug("pipeline config is empty, skipping") @@ -149,7 +149,7 @@ func (e *Executor) Execute( // 通知外部持久化阶段完成状态（用于崩溃恢复） if onStageComplete != nil { - onStageComplete(stageIndex, output) + onStageComplete(stageIndex, result) } // 更新文件列表给下一阶段 @@ -163,13 +163,13 @@ func (e *Executor) Execute( } // 全部成功：执行延迟删除，并让最终结果反映磁盘上的文件 - // 如果上下文已取消，跳过删除以保留所有文件 + // 如果上下文已取消，跳过删除以保留所有文件，返回取消错误让任务标记为 Cancelled if ctx.Ctx.Err() != nil { e.logger.Warn("pipeline cancelled after completion, skipping file cleanup") if len(results) > 0 { results[len(results)-1].OutputFiles = files } - return results, nil + return results, ctx.Ctx.Err() } keptFiles := e.deleteMarkedFiles(files) if len(results) > 0 { From 3dd9f153e574891bc4b1cf82ec023a9d08cf2619 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:40:44 +0800 Subject: [PATCH 15/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E3=80=81=E5=B4=A9=E6=BA=83=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E5=92=8C=E8=BF=9B=E5=BA=A6=E8=AE=A1=E7=AE=97=E5=A4=9A=E4=B8=AA?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RetryTaskFromStage: 合并标记过滤与存在性校验 修复失败/取消任务重试时标记文件被误删导致重试失败和文件泄漏 - RetryTask: 新增 InitialFiles 存在性校验 修复 Completed 任务文件已删除时重试带着幽灵路径执行 - onStageComplete: 持久化完整 StageResult 而非仅 CurrentFiles 修复崩溃恢复后已完成阶段的执行记录（commands/logs）丢失 - UpdateProgress: 复用已有方法替代内联计算 修复 TotalStages 为 0 时整数除零 panic --- src/pipeline/manager.go | 48 +++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index f5d66b4..4643c1e 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -226,11 +226,16 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { } m.broadcastTaskUpdate(task) }, - // 阶段完成后持久化 CurrentFiles，确保崩溃恢复时能拿到正确的中间文件 - func(stageIndex int, outputFiles []FileInfo) { - task.CurrentFiles = outputFiles + // 阶段完成后持久化结果，确保崩溃恢复时阶段记录完整 + func(stageIndex int, result StageResult) { + task.CurrentFiles = result.OutputFiles + // 追加/更新 StageResults，让崩溃恢复后 preservedResults 合并条件成立 + for len(task.StageResults) <= stageIndex { + task.StageResults = append(task.StageResults, StageResult{}) + } + task.StageResults[stageIndex] = result if err := m.store.UpdateTask(ctx, task); err != nil { - logrus.WithError(err).Warn("failed to persist stage output files") + logrus.WithError(err).Warn("failed to persist stage result") } }, ) @@ -359,6 +364,13 @@ func (m *Manager) RetryTask(taskID int64) error { return fmt.Errorf("only failed, cancelled or completed tasks can be retried") } + // 校验初始文件存在（Completed 任务的文件可能已被 deleteMarkedFiles 清理） + for _, f := range task.InitialFiles { + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + return fmt.Errorf("initial file not found, cannot retry (may have been deleted after upload): %s", f.Path) + } + } + // 重置任务状态 task.Status = PipelineStatusPending task.StartedAt = nil @@ -444,18 +456,25 @@ func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error { inputFiles = task.InitialFiles } - // 过滤掉已标记删除/已上传的中间文件（它们在管道完成后已被 deleteMarkedFiles 从磁盘移除） - // 只保留目标阶段实际需要的文件，确保后续 os.Stat 校验和重试执行不会引用已删除的路径 + // 过滤已清理的中间文件，保留仍在磁盘上的文件 + // deleteMarkedFiles 仅在管道全部成功后执行，失败/取消时文件仍在磁盘但带有标记 + // 因此需要结合磁盘实际存在性判断：标记但仍在 → 保留（管道未清理），标记且已删 → 跳过 var activeFiles []FileInfo for _, f := range inputFiles { - if f.Deletable { - continue - } + isDeletable := f.Deletable + isUploaded := false if f.Metadata != nil { if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { - continue + isUploaded = true } } + + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + if isDeletable || isUploaded { + continue // 已被 deleteMarkedFiles 正常清理，跳过 + } + return fmt.Errorf("input file not found, cannot retry: %s", f.Path) + } activeFiles = append(activeFiles, f) } inputFiles = activeFiles @@ -465,13 +484,6 @@ func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error { return fmt.Errorf("no input files available for stage %s, cannot retry", stageName) } - // 验证文件存在于磁盘 - for _, f := range inputFiles { - if _, err := os.Stat(f.Path); os.IsNotExist(err) { - return fmt.Errorf("input file not found, cannot retry: %s", f.Path) - } - } - // 重置任务状态 task.Status = PipelineStatusPending task.StartedAt = nil @@ -483,7 +495,7 @@ func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error { if targetStageIndex < len(task.StageResults) { task.StageResults = task.StageResults[:targetStageIndex] } - task.Progress = (targetStageIndex * 100) / task.TotalStages + task.UpdateProgress() if err := m.store.UpdateTask(m.ctx, task); err != nil { return err From 2437386e302b56b9f46a97f0dceb24276272e185 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:40:52 +0800 Subject: [PATCH 16/52] =?UTF-8?q?fix(pipeline):=20delete=5Fall=20=E6=A0=87?= =?UTF-8?q?=E8=AE=B0=E8=B7=B3=E8=BF=87=E5=B7=B2=E6=98=AF=20Deletable=20?= =?UTF-8?q?=E7=9A=84=E4=B8=AD=E9=97=B4=E4=BA=A7=E7=89=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复 delete_all_after_upload 模式下，中间产物（如 BurnSubtitles 标记的源视频） 被追加 delete_all 标记后，CustomCommandStage 对其误执行自定义命令的问题。 --- src/pipeline/stages/extract_cover.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index af42e4a..a7157dc 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -304,7 +304,13 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } // 删除全部文件（含中间产物）：用 Deletable 标记 + // 跳过已是 Deletable 的中间产物（如 BurnSubtitles 标记的源视频）， + // 避免 delete_all 标记覆盖其"中间产物"身份，导致 CustomCommandStage 误执行 for i := range output { + if output[i].Deletable { + // 中间产物：保留原始 Deletable 标记，不加 delete_all + continue + } output[i].Deletable = true if output[i].Metadata == nil { output[i].Metadata = map[string]any{} From 38068042b064b67dccdf13c6336f2e92695033ec Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:40:57 +0800 Subject: [PATCH 17/52] =?UTF-8?q?fix(openlist):=20=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E6=B8=85=E7=90=86=E8=B7=AF=E5=BE=84=E4=BD=BF=E7=94=A8=20CAS=20?= =?UTF-8?q?=E9=81=BF=E5=85=8D=E8=AF=AF=E6=B8=85=E6=96=B0=20Manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 ClearGlobalManagerIf(expected) 函数，仅当全局指针仍指向入参时才清除。 修复快速「关→开」场景下，旧 Manager 异步 Start 失败误清新 Manager 全局注册的问题。 四个异步清理路径全部改用 CAS 语义，两个同步路径（用户意图）保持无条件清除。 --- src/cmd/bililive/bililive.go | 13 +++++++------ src/pkg/openlist/manager.go | 13 +++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/cmd/bililive/bililive.go b/src/cmd/bililive/bililive.go index f71dd59..9def416 100644 --- a/src/cmd/bililive/bililive.go +++ b/src/cmd/bililive/bililive.go @@ -359,8 +359,8 @@ func main() { bilisentryPkg.Go(func() { if err := openlistManager.Start(rootCtx); err != nil { logger.WithError(err).Error("启动 OpenList 失败，云上传功能将不可用") - // 启动失败时清除全局指针，避免 GetGlobalManager() != nil 永久阻止后续重试 - servers.SetOpenListManager(nil) + // 启动失败时清除全局指针（CAS：仅当仍为当前实例时才清除，避免误清新 Manager） + openlist.ClearGlobalManagerIf(openlistManager) openlistManager = nil } }) @@ -422,7 +422,8 @@ func main() { if err := newMgr.Start(rootCtx); err != nil { logger.WithError(err).Error("重建 OpenList 失败") openlistLifecycleMu.Lock() - servers.SetOpenListManager(nil) + // CAS：仅当全局仍为 newMgr 时才清除 + openlist.ClearGlobalManagerIf(newMgr) openlistManager = nil openlistLifecycleMu.Unlock() } @@ -460,9 +461,9 @@ func main() { bilisentryPkg.Go(func() { if err := mgr.Start(rootCtx); err != nil { logger.WithError(err).Error("动态启动 OpenList 失败，云上传功能将不可用") - // 启动失败时清除全局指针，允许后续配置变更时重试 + // CAS：仅当全局仍为当前实例时才清除，避免误清新 Manager openlistLifecycleMu.Lock() - servers.SetOpenListManager(nil) + openlist.ClearGlobalManagerIf(mgr) openlistLifecycleMu.Unlock() return } @@ -472,7 +473,7 @@ func main() { if cfg := configs.GetCurrentConfig(); cfg == nil || !cfg.OnRecordFinished.CloudUpload.Enable { logger.Info("OpenList 启动完成但云上传已关闭，立即停止") mgr.Stop() - servers.SetOpenListManager(nil) + openlist.ClearGlobalManagerIf(mgr) } }) }) diff --git a/src/pkg/openlist/manager.go b/src/pkg/openlist/manager.go index 28f9993..65a4fa1 100644 --- a/src/pkg/openlist/manager.go +++ b/src/pkg/openlist/manager.go @@ -34,6 +34,19 @@ func SetGlobalManager(m *Manager) { globalManager = m } +// ClearGlobalManagerIf 仅当全局管理器等于 expected 时才清除（CAS 语义） +// 用于异步清理路径，避免误清已被新实例替换的管理器 +// 返回是否成功清除 +func ClearGlobalManagerIf(expected *Manager) bool { + globalMu.Lock() + defer globalMu.Unlock() + if globalManager == expected { + globalManager = nil + return true + } + return false +} + // GetGlobalManager 获取全局 OpenList 管理器 func GetGlobalManager() *Manager { globalMu.RLock() From 9d11c8d2c1086d24b8a5994b4f2e57759c33d122 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:07 +0800 Subject: [PATCH 18/52] =?UTF-8?q?fix(webapp):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=84=E7=90=86=E9=A2=84=E8=A7=88=E4=B8=8E?= =?UTF-8?q?=E5=AE=9E=E9=99=85=E4=B8=8A=E4=BC=A0=E8=A1=8C=E4=B8=BA=E4=B8=8D?= =?UTF-8?q?=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - burn+!burnDelSource 时源视频标记为 uploaded - convert+!deleteFlv 时源视频标记为 uploaded - delAllAfter 改为通用逻辑：保存已上传状态 → 全部删除 → 恢复已上传标记 - delAfter 简化为通用遍历，不再逐文件硬编码 --- .../src/component/config-info/CloudUploadSettings.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index b2e047a..0fedb9e 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -146,6 +146,12 @@ const analyzeConfig = (config: any) => { } else { files['video.flv'].status = 'deleted'; } + } else { + // delete_flv_after_convert=false：源视频保留在 ConvertMp4Stage output 中 + // immediate 模式下已在 pipeline 开头被上传，after_process 模式下也会被上传 + if (files['video.flv'].status === 'kept') { + files['video.flv'].status = 'uploaded'; + } } } From 10e1ad677d8ca98f35a7e4b6065eb309871a45b4 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:36:57 +0800 Subject: [PATCH 19/52] =?UTF-8?q?fix(webapp):=20=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=84=E7=90=86=E9=A2=84=E8=A7=88=E8=A1=A5=E5=85=85=E8=AF=B4?= =?UTF-8?q?=E6=98=8E=E6=96=87=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在预览卡片底部添加说明：不开启删除源文件的开关时，原始视频也会一并上传到云端。 帮助用户理解预览中出现多个视频文件的原因。 --- src/webapp/src/component/config-info/CloudUploadSettings.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 0fedb9e..a24a0f0 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -289,6 +289,11 @@ const FileProcessingPreview: React.FC<{ config: any }> = ({ config }) => { } +
+
💡 说明：开启「转换后删除 FLV」或「烧录后删除源视频」后，只会上传最终成品（MP4 或 MKV）。
+
如果不开启，原始视频也会一并上传到云端（相当于云端保存了两份视频）。
+
上方预览会根据你当前的设置实时显示哪些文件会被上传、删除或保留。
+
); }; From d92070a89cbd9880e014eecaa59cff489d32b41f Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:13:47 +0800 Subject: [PATCH 20/52] =?UTF-8?q?ci(release):=20=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E7=94=9F=E6=88=90=20Release=20=E6=9B=B4=E6=96=B0=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 97c40a5..d9702c4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -130,6 +130,7 @@ jobs: dist/**/*.zip dist/**/*.7z prerelease: ${{ contains(env.GIT_TAG, 'rc') }} + generate_release_notes: true - name: "[Dry-Run] 跳过 GitHub Release 发布" if: ${{ env.IS_DRY_RUN == 'true' }} run: echo "::notice::Dry-run 模式：跳过 GitHub Release 发布（GIT_TAG=$GIT_TAG）" From 61abe70df2c26fe20dc8fffa6a7213eeebdbf3fd Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:03:54 +0800 Subject: [PATCH 21/52] =?UTF-8?q?feat(upload):=20=E6=89=8B=E5=8A=A8?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=94=B9=E7=94=A8=20Pipeline=20=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=EF=BC=8C=E4=B8=8E=E8=87=AA=E5=8A=A8=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E8=A1=8C=E4=B8=BA=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 手动上传（文件管理界面）原来直接调用 OpenList 客户端上传， 路径推断逻辑与自动上传（CloudUploadStage）不一致，且无任务追踪。 改为创建单阶段 cloud_upload PipelineTask，复用 CloudUploadStage 的 完整逻辑：路径模板渲染、上传标记持久化、任务追踪、重试能力。 主要改动： - 新增 EnqueueUploadTask 方法，接收绝对路径列表创建 Pipeline 任务 - 新增 upload_handler.go，通过 Pipeline 系统处理手动上传请求 - 提取 RenderUploadPath 为公共函数，消除 extract_cover.go 中的重复代码 - 上传成功后持久化标记到 metadata DB，文件列表显示已上传标签 - 文件删除/重命名时自动清除/迁移上传标记 - Pipeline 任务错误信息支持多行展示 Co-Authored-By: Kscc --- src/pipeline/executor.go | 15 +++ src/pipeline/manager.go | 91 +++++++++++++ src/pipeline/stages/extract_cover.go | 90 +++++-------- src/pipeline/upload_utils.go | 67 ++++++++++ src/pkg/metadata/store.go | 2 + src/servers/handler.go | 98 +++++++++++++- src/servers/server.go | 3 +- src/servers/upload_handler.go | 125 ++++++++++++++++++ src/webapp/src/component/file-list/index.tsx | 96 +++++++++++++- .../component/pipeline-task-list/index.tsx | 12 +- src/webapp/src/utils/api.ts | 16 +++ 11 files changed, 550 insertions(+), 65 deletions(-) create mode 100644 src/pipeline/upload_utils.go create mode 100644 src/servers/upload_handler.go diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 0949aec..02323be 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/pkg/metadata" bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry" "github.com/sirupsen/logrus" ) @@ -185,6 +187,9 @@ func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { deleteAll := false // 标记是否为 deleteAll 模式 deleteAfter := false // 标记是否为 deleteAfter 模式（仅删除已上传文件） + // 获取输出路径用于计算相对路径（清理 DB 上传标记） + cfg := configs.GetCurrentConfig() + for _, f := range files { // 检查删除模式（CloudUploadStage 标记） if f.Metadata != nil { @@ -212,6 +217,16 @@ func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { } } else { e.logger.Infof("pipeline cleanup: 已删除 %s", f.Path) + // 清除 DB 中的上传标记 + if cfg != nil { + outPutPath, _ := filepath.Abs(cfg.OutPutPath) + absFilePath, _ := filepath.Abs(f.Path) + if relPath, relErr := filepath.Rel(outPutPath, absFilePath); relErr == nil { + if delErr := metadata.GetStore().Delete(context.Background(), metadata.NamespaceUploaded, filepath.ToSlash(relPath)); delErr != nil { + e.logger.Warnf("pipeline cleanup: 清除上传标记失败 %s: %v", f.Path, delErr) + } + } + } } } else { kept = append(kept, f) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 4643c1e..5f72e3d 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -4,14 +4,18 @@ import ( "context" "fmt" "os" + "path/filepath" + "strings" "sync" "time" + "github.com/bililive-go/bililive-go/src/configs" "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry" + "github.com/bililive-go/bililive-go/src/types" "github.com/sirupsen/logrus" ) @@ -584,6 +588,93 @@ type ManagerStats struct { CancelledCount int `json:"cancelled_count"` } +// EnqueueUploadTask 创建手动上传的 Pipeline 任务并入队 +// 行为与自动上传（CloudUploadStage）完全一致：使用相同的路径模板、配置选项 +// 每个文件创建一个独立的单阶段 cloud_upload 任务 +// absPaths: 已校验的绝对路径列表（调用方应先通过 getSafePath 校验） +func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped []string, taskIDs []int64, err error) { + config := configs.GetCurrentConfig() + if config == nil { + return 0, nil, nil, fmt.Errorf("配置未初始化") + } + + cu := config.OnRecordFinished.CloudUpload + if !cu.Enable { + return 0, nil, nil, fmt.Errorf("云上传功能未启用，请先在设置中开启") + } + if cu.StorageName == "" { + return 0, nil, nil, fmt.Errorf("未配置存储名称") + } + + // 构建单阶段 cloud_upload PipelineConfig，与 ConvertLegacyConfig 一致 + uploadConfig := &PipelineConfig{ + Stages: []StageConfig{ + { + Name: StageNameCloudUpload, + Options: map[string]any{ + OptionStorage: cu.StorageName, + OptionPathTemplate: cu.UploadPathTmpl, + OptionDeleteAfter: cu.DeleteAfterUpload, + OptionDeleteAllAfter: cu.DeleteAllAfterUpload, + OptionUploadTiming: string(config.OnRecordFinished.UploadTiming), + OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)}, + }, + }, + }, + } + + outputPath, _ := filepath.Abs(config.OutPutPath) + skipped = []string{} + + for _, absPath := range absPaths { + // 检查文件存在 + info, statErr := os.Stat(absPath) + if statErr != nil { + skipped = append(skipped, filepath.Base(absPath)+" - 文件不存在或无法访问") + continue + } + if info.IsDir() { + skipped = append(skipped, filepath.Base(absPath)+" - 不支持上传文件夹") + continue + } + + // 从相对路径推断 RecordInfo（用于上传路径模板渲染） + relPath, _ := filepath.Rel(outputPath, absPath) + platform, hostName := inferUploadPathInfo(relPath) + + recordInfo := RecordInfo{ + LiveID: types.LiveID("manual-upload"), + Platform: platform, // 用于上传路径模板：{{ .Platform }} + HostName: hostName, // 用于上传路径模板：{{ .HostName }} + RoomName: filepath.Base(absPath), // 显示文件名，便于在任务列表中识别 + StartTime: info.ModTime(), + } + + files := []FileInfo{NewVideoFileInfo(absPath)} + task := NewPipelineTask(recordInfo, uploadConfig, files) + + if enqueueErr := m.EnqueueTask(task); enqueueErr != nil { + skipped = append(skipped, filepath.Base(absPath)+" - 入队失败: "+enqueueErr.Error()) + continue + } + + enqueued++ + taskIDs = append(taskIDs, task.ID) + } + + return enqueued, skipped, taskIDs, nil +} + +// inferUploadPathInfo 从文件相对路径推断平台和主播名 +// 目录结构约定：{Platform}/{HostName}/... +func inferUploadPathInfo(relPath string) (platform, hostName string) { + parts := strings.Split(filepath.ToSlash(relPath), "/") + if len(parts) >= 2 { + return parts[0], parts[1] + } + return "", "" +} + // GetManager 从实例获取管道管理器 func GetManager(inst *instance.Instance) *Manager { if inst == nil || inst.PipelineManager == nil { diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index a7157dc..221fe38 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -1,17 +1,16 @@ package stages import ( - "bytes" "fmt" "os" "path" "path/filepath" "strings" - "text/template" "time" "github.com/bililive-go/bililive-go/src/configs" "github.com/bililive-go/bililive-go/src/pipeline" + "github.com/bililive-go/bililive-go/src/pkg/metadata" "github.com/bililive-go/bililive-go/src/pkg/openlist" "github.com/bililive-go/bililive-go/src/tools" ) @@ -173,6 +172,7 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli var output []pipeline.FileInfo var uploadFailed bool usedPaths := map[string]bool{} // 检测远程路径冲突 + var failureDetails []string // 收集每条失败详情，用于返回给前端 // 上传所有非 Deletable 的视频文件和封面文件 // Deletable 检查已能正确区分中间产物（如 ConvertMp4 标记的原始 FLV） @@ -194,7 +194,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { - s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) + errMsg := fmt.Sprintf("文件不存在: %s", file.Path) + s.logs += errMsg + "\n" + failureDetails = append(failureDetails, errMsg) output = append(output, file) uploadFailed = true continue @@ -203,7 +205,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 渲染目标路径 targetPath := s.renderTargetPath(ctx, file) if targetPath == "" { - s.logs += fmt.Sprintf("无法生成目标路径: %s\n", file.Path) + errMsg := fmt.Sprintf("无法生成目标路径: %s", file.Path) + s.logs += errMsg + "\n" + failureDetails = append(failureDetails, errMsg) output = append(output, file) uploadFailed = true continue @@ -232,6 +236,7 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } // 确保存储名和路径之间没有双斜杠 fullRemotePath := "/" + s.storageName + remotePath + fullRemotePath = strings.ReplaceAll(fullRemotePath, "//", "/") // 确保远程目录存在（递归创建） // 使用 path.Dir 而非 filepath.Dir，因为远程路径始终用正斜杠 @@ -250,7 +255,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli }) if err != nil { - s.logs += fmt.Sprintf("上传失败: %s -> %s/%s - %s\n", fileName, s.storageName, targetPath, err.Error()) + errMsg := fmt.Sprintf("上传失败: %s -> %s/%s - %s", fileName, s.storageName, targetPath, err.Error()) + s.logs += errMsg + "\n" + failureDetails = append(failureDetails, errMsg) ctx.Logger.Errorf("上传失败: %s - %v", file.Path, err) // 上传失败，保留文件 output = append(output, file) @@ -262,6 +269,17 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli ctx.Logger.Infof("文件上传成功: %s -> %s/%s", fileName, s.storageName, targetPath) uploadedIndices[len(output)] = true // 记录上传成功的文件索引 output = append(output, file) + + // 持久化上传标记（统一用正斜杠存储，与 URL 路径一致） + if cfg := configs.GetCurrentConfig(); cfg != nil { + outPutPath, _ := filepath.Abs(cfg.OutPutPath) + absFilePath, _ := filepath.Abs(file.Path) + if relPath, relErr := filepath.Rel(outPutPath, absFilePath); relErr == nil { + if markErr := metadata.GetStore().Set(ctx.Ctx, metadata.NamespaceUploaded, filepath.ToSlash(relPath), "1"); markErr != nil { + ctx.Logger.Warnf("保存上传标记失败: %v", markErr) + } + } + } } // 全部上传成功后，根据配置标记文件为可删除（由 Executor 在管道全部成功后统一删除） @@ -337,8 +355,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } // 任意上传失败，返回错误（让 Executor 将任务标记为 failed） + // 将每条失败详情拼入 error message，前端可直接展示具体原因 if uploadFailed { - return output, fmt.Errorf("部分文件上传失败，详见日志") + return output, fmt.Errorf("部分文件上传失败:\n%s", strings.Join(failureDetails, "\n")) } return output, nil @@ -356,29 +375,12 @@ func (s *CloudUploadStage) matchFileType(fileType pipeline.FileType) bool { // renderTargetPath 渲染目标路径 func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo) string { - if s.pathTemplate == "" { - // 默认路径：/录播归档/{平台}/{主播名}/{文件名} - return fmt.Sprintf("/录播归档/%s/%s/%s", - ctx.RecordInfo.Platform, - ctx.RecordInfo.HostName, - filepath.Base(file.Path), - ) - } - - // 获取扩展名 ext := filepath.Ext(file.Path) if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } - // 模板数据 - data := struct { - Platform string - HostName string - RoomName string - FileName string - Ext string - }{ + data := pipeline.UploadPathData{ Platform: ctx.RecordInfo.Platform, HostName: ctx.RecordInfo.HostName, RoomName: ctx.RecordInfo.RoomName, @@ -386,44 +388,14 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file Ext: ext, } - // 使用 Go 模板引擎 - funcMap := template.FuncMap{ - "date": func(format string, t time.Time) string { - return t.Format(format) - }, - "now": func() time.Time { - return ctx.RecordInfo.StartTime - }, - "trimSuffix": func(suffix, s string) string { - return strings.TrimSuffix(s, suffix) - }, - "ext": func(path string) string { - return filepath.Ext(path) - }, - "filenameFilter": func(s string) string { - // 过滤文件名中的非法字符 - replacer := strings.NewReplacer( - "/", "_", "\\", "_", ":", "_", "*", "_", - "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", - ) - return replacer.Replace(s) - }, - } - - tmpl, err := template.New("path").Funcs(funcMap).Parse(s.pathTemplate) + path, err := pipeline.RenderUploadPath(s.pathTemplate, data, func() time.Time { + return ctx.RecordInfo.StartTime + }) if err != nil { - // 模板语法错误，返回空让调用方跳过上传，避免上传到错误路径 - ctx.Logger.Errorf("上传路径模板解析失败: %v", err) + ctx.Logger.Errorf("%v", err) return "" } - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, data); err != nil { - ctx.Logger.Errorf("上传路径模板执行失败: %v", err) - return "" - } - - return buf.String() + return path } func (s *CloudUploadStage) GetCommands() []string { diff --git a/src/pipeline/upload_utils.go b/src/pipeline/upload_utils.go new file mode 100644 index 0000000..d6c0b88 --- /dev/null +++ b/src/pipeline/upload_utils.go @@ -0,0 +1,67 @@ +package pipeline + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" + "text/template" + "time" +) + +// UploadPathData 上传路径模板数据 +type UploadPathData struct { + Platform string + HostName string + RoomName string + FileName string + Ext string +} + +// DefaultUploadPathFuncs 返回上传路径模板的默认函数集 +// nowFn 返回当前时间（可从 RecordInfo.StartTime 或 time.Now() 传入） +func DefaultUploadPathFuncs(nowFn func() time.Time) template.FuncMap { + return template.FuncMap{ + "date": func(format string, t time.Time) string { + return t.Format(format) + }, + "now": nowFn, + "trimSuffix": func(suffix, s string) string { + return strings.TrimSuffix(s, suffix) + }, + "ext": func(path string) string { + return filepath.Ext(path) + }, + "filenameFilter": func(s string) string { + replacer := strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", + "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", + ) + return replacer.Replace(s) + }, + } +} + +// RenderUploadPath 渲染上传目标路径 +// pathTemplate 为空时使用默认路径 /录播归档/{Platform}/{HostName}/{FileName} +// 返回空字符串表示模板解析失败 +func RenderUploadPath(pathTemplate string, data UploadPathData, nowFn func() time.Time) (string, error) { + if pathTemplate == "" { + path := fmt.Sprintf("/录播归档/%s/%s/%s", data.Platform, data.HostName, data.FileName) + return path, nil + } + + funcMap := DefaultUploadPathFuncs(nowFn) + + tmpl, err := template.New("path").Funcs(funcMap).Parse(pathTemplate) + if err != nil { + return "", fmt.Errorf("上传路径模板解析失败: %w", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("上传路径模板执行失败: %w", err) + } + + return buf.String(), nil +} diff --git a/src/pkg/metadata/store.go b/src/pkg/metadata/store.go index d1c377e..7912307 100644 --- a/src/pkg/metadata/store.go +++ b/src/pkg/metadata/store.go @@ -206,6 +206,8 @@ const ( NamespaceUpdate = "update" // NamespaceMigration 数据库迁移状态 NamespaceMigration = "migration" + // NamespaceUploaded 已上传文件标记（key=相对路径, value="1"） + NamespaceUploaded = "uploaded" ) // 预定义的键常量 diff --git a/src/servers/handler.go b/src/servers/handler.go index a620efd..2cfd0b9 100644 --- a/src/servers/handler.go +++ b/src/servers/handler.go @@ -35,6 +35,7 @@ import ( applog "github.com/bililive-go/bililive-go/src/log" "github.com/bililive-go/bililive-go/src/pkg/livelogger" "github.com/bililive-go/bililive-go/src/pkg/memstats" + "github.com/bililive-go/bililive-go/src/pkg/metadata" "github.com/bililive-go/bililive-go/src/pkg/ratelimit" "github.com/bililive-go/bililive-go/src/pkg/utils" "github.com/bililive-go/bililive-go/src/recorders" @@ -2419,6 +2420,7 @@ func getFileInfo(writer http.ResponseWriter, r *http.Request) { LastModified int64 `json:"last_modified"` Size int64 `json:"size"` SubtitleFile string `json:"subtitle_file,omitempty"` + Uploaded bool `json:"uploaded,omitempty"` } // First pass: separate ASS files and build base-name -> ASS file map @@ -2461,6 +2463,14 @@ func getFileInfo(writer http.ResponseWriter, r *http.Request) { if assName, ok := assFiles[baseName]; ok { jf.SubtitleFile = assName } + // Check if this file has been uploaded to cloud + relPath := fe.dir.Name() + if path != "" { + relPath = path + "/" + fe.dir.Name() // key 统一用正斜杠 + } + if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, relPath); err == nil && val != "" { + jf.Uploaded = true + } } jsonFiles = append(jsonFiles, jf) } @@ -2580,6 +2590,30 @@ func renameFile(writer http.ResponseWriter, r *http.Request) { } } + // 迁移上传标记（key 统一用正斜杠） + oldRel := path + newRelPath, _ := filepath.Rel(base, newAbsPath) + newRel := filepath.ToSlash(newRelPath) + if info.IsDir() { + // 目录重命名：迁移该目录下所有文件的上传标记 + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + oldPrefix := oldRel + "/" + for key, val := range allMarks { + if strings.HasPrefix(key, oldPrefix) { + suffix := key[len(oldPrefix):] + newKey := newRel + "/" + suffix + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newKey, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + // 文件重命名：迁移单个上传标记 + if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, oldRel); err == nil { + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newRel, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, oldRel) + } + } + writeJSON(writer, commonResp{Data: "OK"}) } @@ -2599,8 +2633,14 @@ func deleteFile(writer http.ResponseWriter, r *http.Request) { return } + // 记录是否为目录（删除前检查） + isDir := false + if info, err := os.Stat(absPath); err == nil { + isDir = info.IsDir() + } + // 删除关联的 ASS 弹幕文件 - if info, err := os.Stat(absPath); err == nil && !info.IsDir() { + if !isDir { assPath := strings.TrimSuffix(absPath, filepath.Ext(absPath)) + ".ass" if _, err := os.Stat(assPath); err == nil { os.Remove(assPath) @@ -2612,6 +2652,20 @@ func deleteFile(writer http.ResponseWriter, r *http.Request) { return } + // 清除上传标记 + if isDir { + // 目录：清除该目录下所有文件的上传标记 + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + prefix := path + "/" // key 统一用正斜杠 + for key := range allMarks { + if strings.HasPrefix(key, prefix) { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, path) + } + writeJSON(writer, commonResp{Data: "OK"}) } @@ -2698,6 +2752,28 @@ func batchRenameFiles(writer http.ResponseWriter, r *http.Request) { os.Rename(oldAss, newAss) } } + + // 迁移上传标记（key 统一用正斜杠） + oldRel := path + newRelPath, _ := filepath.Rel(base, newAbsPath) + newRel := filepath.ToSlash(newRelPath) + if info.IsDir() { + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + oldPrefix := oldRel + "/" + for key, val := range allMarks { + if strings.HasPrefix(key, oldPrefix) { + suffix := key[len(oldPrefix):] + newKey := newRel + "/" + suffix + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newKey, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, oldRel); err == nil { + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newRel, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, oldRel) + } + } } } @@ -2734,8 +2810,14 @@ func batchDeleteFiles(writer http.ResponseWriter, r *http.Request) { continue } + // 记录是否为目录（删除前检查） + isDir := false + if info, err := os.Stat(absPath); err == nil { + isDir = info.IsDir() + } + // 删除关联的 ASS 弹幕文件 - if info, err := os.Stat(absPath); err == nil && !info.IsDir() { + if !isDir { assPath := strings.TrimSuffix(absPath, filepath.Ext(absPath)) + ".ass" if _, err := os.Stat(assPath); err == nil { os.Remove(assPath) @@ -2745,6 +2827,18 @@ func batchDeleteFiles(writer http.ResponseWriter, r *http.Request) { if err := os.RemoveAll(absPath); err != nil { results = append(results, Result{Path: path, Success: false, Message: translateOSError(err)}) } else { + // 清除上传标记 + if isDir { + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + prefix := path + "/" // key 统一用正斜杠 + for key := range allMarks { + if strings.HasPrefix(key, prefix) { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, path) + } results = append(results, Result{Path: path, Success: true, Message: "成功"}) } } diff --git a/src/servers/server.go b/src/servers/server.go index e24191b..fcf7c4a 100644 --- a/src/servers/server.go +++ b/src/servers/server.go @@ -153,10 +153,11 @@ func initMux(ctx context.Context) *mux.Router { apiRoute.HandleFunc("/openlist/status", getOpenListStatus).Methods("GET") apiRoute.HandleFunc("/openlist/check-storage", checkOpenListStorageHealth).Methods("GET") - // Pipeline 任务路由 + // Pipeline 任务路由 + 手动上传路由 inst := instance.GetInstance(ctx) if pm := pipeline.GetManager(inst); pm != nil { RegisterPipelineHandlers(apiRoute, pm) + RegisterUploadHandlers(apiRoute, pm) } // OSRP 开放直播录制协议路由 diff --git a/src/servers/upload_handler.go b/src/servers/upload_handler.go new file mode 100644 index 0000000..f1dc1a6 --- /dev/null +++ b/src/servers/upload_handler.go @@ -0,0 +1,125 @@ +package servers + +import ( + "encoding/json" + "net/http" + "path/filepath" + + "github.com/gorilla/mux" + + "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/pipeline" +) + +// uploadResult 手动上传的响应结构 +type uploadResult struct { + Enqueued int `json:"enqueued"` + Skipped []string `json:"skipped"` + TaskIDs []int64 `json:"task_ids"` +} + +// RegisterUploadHandlers 注册手动上传相关的 HTTP 处理器 +// 通过 Pipeline 系统执行上传，与自动上传行为完全一致 +func RegisterUploadHandlers(r *mux.Router, pm *pipeline.Manager) { + if pm == nil { + return + } + + r.HandleFunc("/file/upload", makeUploadFileHandler(pm)).Methods("POST") + r.HandleFunc("/batch/file/upload", makeBatchUploadFilesHandler(pm)).Methods("POST") +} + +// makeUploadFileHandler 单文件上传到云盘（通过 Pipeline） +func makeUploadFileHandler(pm *pipeline.Manager) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Path string `json:"path"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, commonResp{ErrMsg: "请求参数错误"}) + return + } + if req.Path == "" { + writeJSON(w, commonResp{ErrMsg: "文件路径不能为空"}) + return + } + + // 路径校验（复用 handler.go 中的 getSafePath） + cfg := configs.GetCurrentConfig() + if cfg == nil { + writeJSON(w, commonResp{ErrMsg: "配置未初始化"}) + return + } + absPath, err := getSafePath(cfg.OutPutPath, req.Path) + if err != nil { + writeJSON(w, commonResp{ErrMsg: "无效路径: " + err.Error()}) + return + } + + enqueued, skipped, taskIDs, err := pm.EnqueueUploadTask([]string{absPath}) + if err != nil { + writeJSON(w, commonResp{ErrMsg: err.Error()}) + return + } + + writeJSON(w, commonResp{Data: uploadResult{ + Enqueued: enqueued, + Skipped: skipped, + TaskIDs: taskIDs, + }}) + } +} + +// makeBatchUploadFilesHandler 批量上传文件到云盘（通过 Pipeline） +func makeBatchUploadFilesHandler(pm *pipeline.Manager) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Paths []string `json:"paths"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, commonResp{ErrMsg: "请求参数错误"}) + return + } + + // 批量路径校验 + cfg := configs.GetCurrentConfig() + if cfg == nil { + writeJSON(w, commonResp{ErrMsg: "配置未初始化"}) + return + } + + absPaths := make([]string, 0, len(req.Paths)) + skipped := []string{} + for _, p := range req.Paths { + absPath, err := getSafePath(cfg.OutPutPath, p) + if err != nil { + skipped = append(skipped, filepath.Base(p)+" - 路径越权") + continue + } + absPaths = append(absPaths, absPath) + } + + // 如果所有路径都校验失败，直接返回 + if len(absPaths) == 0 { + writeJSON(w, commonResp{Data: uploadResult{ + Enqueued: 0, + Skipped: skipped, + TaskIDs: []int64{}, + }}) + return + } + + enqueued, moreSkipped, taskIDs, err := pm.EnqueueUploadTask(absPaths) + if err != nil { + writeJSON(w, commonResp{ErrMsg: err.Error()}) + return + } + skipped = append(skipped, moreSkipped...) + + writeJSON(w, commonResp{Data: uploadResult{ + Enqueued: enqueued, + Skipped: skipped, + TaskIDs: taskIDs, + }}) + } +} diff --git a/src/webapp/src/component/file-list/index.tsx b/src/webapp/src/component/file-list/index.tsx index ef5432d..3d24a05 100644 --- a/src/webapp/src/component/file-list/index.tsx +++ b/src/webapp/src/component/file-list/index.tsx @@ -11,7 +11,9 @@ import { // @ts-ignore EditOutlined, // @ts-ignore - DeleteOutlined + DeleteOutlined, + // @ts-ignore + CloudUploadOutlined } from "@ant-design/icons"; import { Link, useNavigate, useParams } from "react-router-dom"; import Utils from "../../utils/common"; @@ -479,6 +481,7 @@ type CurrentFolderFile = { last_modified: number; size: number; subtitle_file?: string; + uploaded?: boolean; } const FileList: React.FC = () => { @@ -780,6 +783,71 @@ const FileList: React.FC = () => { .catch(err => message.error("删除失败: " + err)); }; + const handleUpload = (record: CurrentFolderFile, e?: React.MouseEvent) => { + if (e) e.stopPropagation(); + let fullPath = record.name; + if (pathParam) { + fullPath = pathParam + "/" + record.name; + } + + message.loading({ content: `正在上传 ${record.name}...`, key: "upload" }); + api.uploadFile(fullPath) + .then((rsp: any) => { + if (rsp.data) { + const data = rsp.data; + if (data.enqueued > 0) { + message.success({ content: `已入队上传任务: ${record.name}`, key: "upload" }); + } else if (data.skipped && data.skipped.length > 0) { + message.error({ content: data.skipped[0] || "上传失败", key: "upload" }); + } else { + message.error({ content: rsp.err_msg || "上传失败", key: "upload" }); + } + } else { + message.error({ content: rsp.err_msg || "上传失败", key: "upload" }); + } + }) + .catch((err: any) => message.error({ content: "上传失败: " + (err.message || err), key: "upload" })); + }; + + const handleBatchUpload = () => { + if (selectedRowKeys.length === 0) return; + const paths = selectedRowKeys.map(key => { + const fileName = key.toString(); + return pathParam ? `${pathParam}/${fileName}` : fileName; + }); + + message.loading({ content: `正在上传 ${paths.length} 个文件...`, key: "batchUpload" }); + api.batchUploadFiles(paths) + .then((rsp: any) => { + if (rsp.data) { + const data = rsp.data; + const enqueued = data.enqueued || 0; + const skipped = data.skipped || []; + const failCount = skipped.length; + if (failCount === 0) { + message.success({ content: `全部 ${enqueued} 个文件已入队上传`, key: "batchUpload" }); + } else if (enqueued > 0) { + message.warning({ content: `已入队 ${enqueued} 个，跳过 ${failCount} 个`, key: "batchUpload" }); + Modal.info({ + title: '部分文件已跳过', + content: ( +
+ {skipped.map((s: string, i: number) => ( +
{s}
+ ))} +
+ ), + }); + } else { + message.error({ content: skipped[0] || "批量上传失败", key: "batchUpload" }); + } + } else { + message.error({ content: rsp.err_msg || "批量上传失败", key: "batchUpload" }); + } + }) + .catch((err: any) => message.error({ content: "批量上传失败: " + (err.message || err), key: "batchUpload" })); + }; + const handleBatchDelete = () => { if (selectedRowKeys.length === 0) return; const paths = selectedRowKeys.map(key => { @@ -1153,6 +1221,11 @@ const FileList: React.FC = () => { 无字幕 )} + {record.uploaded && ( + + 已上传 + + )} ); } @@ -1194,6 +1267,18 @@ const FileList: React.FC = () => { > 重命名 + {!record.is_folder && ( + + )} handleDelete(record)} @@ -1401,6 +1486,15 @@ const FileList: React.FC = () => { ); })()} + { {result.error_message && ( + {result.error_message} + + } type="error" style={{ marginBottom: 8 }} /> @@ -574,7 +578,11 @@ class PipelineTaskList extends Component { {task.error_message && ( + {task.error_message} + + } type="error" /> )} diff --git a/src/webapp/src/utils/api.ts b/src/webapp/src/utils/api.ts index 183fccd..a39ff32 100644 --- a/src/webapp/src/utils/api.ts +++ b/src/webapp/src/utils/api.ts @@ -170,6 +170,22 @@ class API { return utils.requestPost(`${BASE_URL}/pipeline/batch-burn`, { paths }); } + /** + * 上传单个文件到云盘 + * @param path 文件路径 + */ + uploadFile(path: string) { + return utils.requestPost(`${BASE_URL}/file/upload`, { path }); + } + + /** + * 批量上传文件到云盘 + * @param paths 文件路径列表 + */ + batchUploadFiles(paths: string[]) { + return utils.requestPost(`${BASE_URL}/batch/file/upload`, { paths }); + } + /** * 获取Cookie列表 */ From 2b75cd31f94115109e5361cc47c9601a94c9a731 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:25:35 +0800 Subject: [PATCH 22/52] =?UTF-8?q?fix(upload):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=89=8B=E5=8A=A8=E4=B8=8A=E4=BC=A0=20RecordInfo=20=E6=8E=A8?= =?UTF-8?q?=E6=96=AD=E3=80=81=E8=B7=AF=E5=BE=84=E6=A8=A1=E6=9D=BF=E6=B1=A1?= =?UTF-8?q?=E6=9F=93=E5=92=8C=20metadata=20nil=20panic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. RecordInfo 新增 DisplayName 字段，手动上传 RoomName 改为空字符串， 避免文件名污染 {{.RoomName}} 路径模板渲染（修复简洁归档/按房间归档预设） 2. inferUploadPathInfo 改为两级策略：优先从文件名方括号解析 HostName， 回退到目录结构推断，自定义 OutputTmpl 时推断更准确 3. 手动上传 OptionUploadTiming 硬编码 after_process，不继承全局配置， 确保 delete_after_upload 在 immediate 模式下也能正常生效 4. metadata Store 所有方法添加 nil receiver 检查，Init 失败时 GetStore() 返回 nil 不再导致 panic，降级为静默跳过标记读写 --- src/pipeline/manager.go | 41 ++++++++++++++----- src/pipeline/types.go | 11 ++--- src/pkg/metadata/store.go | 17 +++++++- .../component/pipeline-task-list/index.tsx | 5 ++- 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 5f72e3d..1f7fab1 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "sync" "time" @@ -616,7 +617,7 @@ func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped [] OptionPathTemplate: cu.UploadPathTmpl, OptionDeleteAfter: cu.DeleteAfterUpload, OptionDeleteAllAfter: cu.DeleteAllAfterUpload, - OptionUploadTiming: string(config.OnRecordFinished.UploadTiming), + OptionUploadTiming: "after_process", // 手动上传无后续处理阶段，始终允许删除标记（不继承全局 upload_timing） OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)}, }, }, @@ -643,11 +644,12 @@ func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped [] platform, hostName := inferUploadPathInfo(relPath) recordInfo := RecordInfo{ - LiveID: types.LiveID("manual-upload"), - Platform: platform, // 用于上传路径模板：{{ .Platform }} - HostName: hostName, // 用于上传路径模板：{{ .HostName }} - RoomName: filepath.Base(absPath), // 显示文件名，便于在任务列表中识别 - StartTime: info.ModTime(), + LiveID: types.LiveID("manual-upload"), + Platform: platform, // 用于上传路径模板：{{ .Platform }} + HostName: hostName, // 用于上传路径模板：{{ .HostName }} + RoomName: "", // 手动上传无房间名，不影响路径模板 + DisplayName: filepath.Base(absPath), // 任务列表展示文件名 + StartTime: info.ModTime(), } files := []FileInfo{NewVideoFileInfo(absPath)} @@ -665,14 +667,31 @@ func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped [] return enqueued, skipped, taskIDs, nil } +// bracketPattern 匹配文件名中的 [xxx] 模式 +var bracketPattern = regexp.MustCompile(`\[([^\]]*)\]`) + // inferUploadPathInfo 从文件相对路径推断平台和主播名 -// 目录结构约定：{Platform}/{HostName}/... +// 两级策略：优先从文件名解析（[date][HostName][RoomName].flv），回退到目录结构推断 func inferUploadPathInfo(relPath string) (platform, hostName string) { - parts := strings.Split(filepath.ToSlash(relPath), "/") - if len(parts) >= 2 { - return parts[0], parts[1] + slashPath := filepath.ToSlash(relPath) + + // 策略 1：从文件名解析 HostName（文件名格式通常是 [date][HostName][RoomName].flv） + fileName := filepath.Base(slashPath) + brackets := bracketPattern.FindAllStringSubmatch(fileName, -1) + if len(brackets) >= 2 { + hostName = brackets[1][1] // 第二个方括号通常是主播名 } - return "", "" + + // 策略 2：从目录结构推断 Platform，以及作为 HostName 的兜底 + parts := strings.Split(slashPath, "/") + if len(parts) >= 2 { + platform = parts[0] + if hostName == "" { + hostName = parts[1] + } + } + + return platform, hostName } // GetManager 从实例获取管道管理器 diff --git a/src/pipeline/types.go b/src/pipeline/types.go index 6da83df..b2a3e3e 100644 --- a/src/pipeline/types.go +++ b/src/pipeline/types.go @@ -55,11 +55,12 @@ func NewCoverFileInfo(path, sourcePath string) FileInfo { // RecordInfo 录制信息 type RecordInfo struct { - LiveID types.LiveID `json:"live_id"` - Platform string `json:"platform"` - HostName string `json:"host_name"` - RoomName string `json:"room_name"` - StartTime time.Time `json:"start_time"` + LiveID types.LiveID `json:"live_id"` + Platform string `json:"platform"` + HostName string `json:"host_name"` + RoomName string `json:"room_name"` + DisplayName string `json:"display_name,omitempty"` // 任务列表展示用，不影响路径渲染 + StartTime time.Time `json:"start_time"` } // NewRecordInfo 从 live.Info 创建录制信息 diff --git a/src/pkg/metadata/store.go b/src/pkg/metadata/store.go index 7912307..ddb7fe0 100644 --- a/src/pkg/metadata/store.go +++ b/src/pkg/metadata/store.go @@ -92,7 +92,7 @@ func Close() error { storeMu.Lock() defer storeMu.Unlock() - if globalStore == nil { + if globalStore == nil || globalStore.db == nil { return nil } @@ -103,6 +103,9 @@ func Close() error { // Get 从指定命名空间获取值 func (s *Store) Get(ctx context.Context, namespace, key string) (string, error) { + if s == nil { + return "", nil + } s.mu.RLock() defer s.mu.RUnlock() @@ -123,6 +126,9 @@ func (s *Store) Get(ctx context.Context, namespace, key string) (string, error) // Set 在指定命名空间设置值 func (s *Store) Set(ctx context.Context, namespace, key, value string) error { + if s == nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() @@ -142,6 +148,9 @@ func (s *Store) Set(ctx context.Context, namespace, key, value string) error { // Delete 从指定命名空间删除键 func (s *Store) Delete(ctx context.Context, namespace, key string) error { + if s == nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() @@ -157,6 +166,9 @@ func (s *Store) Delete(ctx context.Context, namespace, key string) error { // GetAll 获取指定命名空间的所有键值对 func (s *Store) GetAll(ctx context.Context, namespace string) (map[string]string, error) { + if s == nil { + return nil, nil + } s.mu.RLock() defer s.mu.RUnlock() @@ -183,6 +195,9 @@ func (s *Store) GetAll(ctx context.Context, namespace string) (map[string]string // DeleteNamespace 删除整个命名空间 func (s *Store) DeleteNamespace(ctx context.Context, namespace string) error { + if s == nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() diff --git a/src/webapp/src/component/pipeline-task-list/index.tsx b/src/webapp/src/component/pipeline-task-list/index.tsx index 73d2b5e..9ce68d1 100644 --- a/src/webapp/src/component/pipeline-task-list/index.tsx +++ b/src/webapp/src/component/pipeline-task-list/index.tsx @@ -36,6 +36,7 @@ interface RecordInfo { platform: string; host_name: string; room_name: string; + display_name?: string; start_time: string; } @@ -563,7 +564,7 @@ class PipelineTaskList extends Component {
直播间： - {task.record_info.room_name} ({task.record_info.host_name} - {task.record_info.platform}) + {task.record_info.display_name || task.record_info.room_name} ({task.record_info.host_name} - {task.record_info.platform})
)} @@ -642,7 +643,7 @@ class PipelineTaskList extends Component { width: 200, render: (_, record: PipelineTask) => ( - {record.record_info?.room_name || '-'} + {record.record_info?.display_name || record.record_info?.room_name || '-'} {record.record_info?.platform && ( {record.record_info.platform} )} From 3699d798a3ed24c864ff7e8cd7a2ecac16f67d07 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:13:09 +0800 Subject: [PATCH 23/52] =?UTF-8?q?fix(notify):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=BD=95=E5=88=B6=E6=91=98=E8=A6=81=E9=80=9A=E7=9F=A5=E5=9C=A8?= =?UTF-8?q?=20Pipeline=20=E5=A4=84=E7=90=86=E5=89=8D=E5=8F=91=E9=80=81?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=96=87=E4=BB=B6=E5=88=97=E8=A1=A8=E4=B8=8D?= =?UTF-8?q?=E5=87=86=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 录制摘要通知原来在 recorder.run() 退出时立即发送，此时 Pipeline（转码、修复、上传等） 尚未执行。开启「上传后删除」时，通知中的文件实际已被删除，消息严重失真。 修复方案： - recorder 中添加 pipelineEnqueued 标志，Pipeline 任务入队后跳过摘要推送 - Pipeline Manager 在任务完成/失败后发送摘要，反映最终文件状态 - FileInfo 添加 Size 字段，创建时预存文件大小，确保文件删除后仍可构建摘要 - 提取 sendToAllChannels 消除通道推送重复代码 - 新增 buildUploadedSummaryBody 构造「已上传到云端」场景的消息体 Co-Authored-By: Kscc --- src/notify/send.go | 62 +++++++++++++++++++++++++++ src/pipeline/manager.go | 90 +++++++++++++++++++++++++++++++++++++++ src/pipeline/types.go | 10 ++++- src/recorders/recorder.go | 11 +++++ 4 files changed, 171 insertions(+), 2 deletions(-) diff --git a/src/notify/send.go b/src/notify/send.go index 14db3c4..4175205 100644 --- a/src/notify/send.go +++ b/src/notify/send.go @@ -250,6 +250,29 @@ func buildRecordingSummaryMessage(hostName, platform string, files []RecordingFi return } +// buildUploadedSummaryBody 构造"已上传到云端"场景的录制摘要消息体 +// 与 buildRecordingSummaryMessage 类似，但标注文件已上传、不显示磁盘空间 +func buildUploadedSummaryBody(platform string, files []RecordingFileDetail) string { + const maxDisplayFiles = 30 + + var sb strings.Builder + fmt.Fprintf(&sb, "平台：%s\n", platform) + fmt.Fprintf(&sb, "录制文件：%d 个（已上传到云端）\n", len(files)) + var totalSize int64 + for i, f := range files { + totalSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(files) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(files)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "总大小：%s\n", formatFileSize(totalSize)) + fmt.Fprintf(&sb, "本地文件已清理") + return sb.String() +} + // SendRecordingSummary 录制结束后发送录制文件摘要通知 // outputPath 为录制输出路径，用于获取剩余磁盘空间 func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform string, files []RecordingFileDetail, outputPath string) { @@ -262,7 +285,46 @@ func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform stri } title, body := buildRecordingSummaryMessage(hostName, platform, files, outputPath) + sendToAllChannels(cfg, logger, title, body) +} +// SendPipelineRecordingSummary Pipeline 完成后发送录制摘要通知 +// originalFiles: 原始录制文件列表（allUploadedAndDeleted 时用于显示） +// finalFiles: Pipeline 处理后的最终文件列表 +// allUploadedAndDeleted: 所有文件已上传并删除（显示上传状态而非磁盘文件） +func SendPipelineRecordingSummary( + logger *livelogger.LiveLogger, + hostName, platform string, + originalFiles, finalFiles []RecordingFileDetail, + outputPath string, + allUploadedAndDeleted bool, +) { + cfg := configs.GetCurrentConfig() + if cfg == nil || !cfg.Notify.SendRecordingSummary { + return + } + + var title, body string + if allUploadedAndDeleted { + // 所有文件已上传并删除：显示原始文件列表 + 上传状态 + if len(originalFiles) == 0 { + return + } + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildUploadedSummaryBody(platform, originalFiles) + } else { + // 有文件保留或 Pipeline 失败：显示最终文件列表 + if len(finalFiles) == 0 { + return + } + title, body = buildRecordingSummaryMessage(hostName, platform, finalFiles, outputPath) + } + + sendToAllChannels(cfg, logger, title, body) +} + +// sendToAllChannels 向所有已启用的通知通道推送摘要消息 +func sendToAllChannels(cfg *configs.Config, logger *livelogger.LiveLogger, title, body string) { // Telegram if cfg.Notify.Telegram.Enable { msg := fmt.Sprintf("%s\n%s", title, body) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 1f7fab1..53f4969 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -13,6 +13,7 @@ import ( "github.com/bililive-go/bililive-go/src/configs" "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" + "github.com/bililive-go/bililive-go/src/notify" "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry" @@ -277,6 +278,9 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { // 广播任务状态变化 m.broadcastTaskUpdate(task) + + // 发送录制摘要通知（仅对录制任务，非手动上传） + m.sendRecordingSummaryIfNeeded(task) } // broadcastTaskUpdate 广播任务更新事件 @@ -286,6 +290,92 @@ func (m *Manager) broadcastTaskUpdate(task *PipelineTask) { } } +// sendRecordingSummaryIfNeeded 在 Pipeline 任务完成后发送录制摘要通知 +// 仅对录制任务（LiveID 非 "manual-upload"）发送，手动上传不发通知 +func (m *Manager) sendRecordingSummaryIfNeeded(task *PipelineTask) { + // 跳过手动上传任务 + if task.RecordInfo.LiveID == "manual-upload" { + return + } + + // 仅在完成或失败时发送 + if task.Status != PipelineStatusCompleted && task.Status != PipelineStatusFailed { + return + } + + // 获取输出路径（用于查询剩余磁盘空间） + outputPath := "" + if len(task.InitialFiles) > 0 { + outputPath = filepath.Dir(task.InitialFiles[0].Path) + } + + logger := livelogger.New(livelogger.DefaultBufferSize, logrus.Fields{ + "platform": task.RecordInfo.Platform, + "host": task.RecordInfo.HostName, + }) + + switch task.Status { + case PipelineStatusCompleted: + finalFiles := task.CurrentFiles // deleteMarkedFiles 后的保留文件 + + if len(finalFiles) == 0 { + // 所有文件已上传并删除：从 InitialFiles 构建原始文件列表（Size 已预存） + originalDetails := fileInfoToDetails(task.InitialFiles) + notify.SendPipelineRecordingSummary( + logger, + task.RecordInfo.HostName, + task.RecordInfo.Platform, + originalDetails, // originalFiles（显示用） + nil, // finalFiles + outputPath, + true, // allUploadedAndDeleted + ) + } else { + // 有文件保留：显示最终文件列表 + fileDetails := fileInfoToDetails(finalFiles) + notify.SendPipelineRecordingSummary( + logger, + task.RecordInfo.HostName, + task.RecordInfo.Platform, + nil, // originalFiles（不需要） + fileDetails, // finalFiles + outputPath, + false, + ) + } + + case PipelineStatusFailed: + // Pipeline 失败：文件仍在磁盘，用原始文件列表 + fileDetails := fileInfoToDetails(task.InitialFiles) + if len(fileDetails) > 0 { + notify.SendPipelineRecordingSummary( + logger, + task.RecordInfo.HostName, + task.RecordInfo.Platform, + nil, // originalFiles（不需要） + fileDetails, // finalFiles（失败时=原始文件） + outputPath, + false, + ) + } + } +} + +// fileInfoToDetails 将 FileInfo 切片转换为 RecordingFileDetail 切片 +// 使用 FileInfo 中预存的 Size，无需再次 stat 文件 +func fileInfoToDetails(files []FileInfo) []notify.RecordingFileDetail { + details := make([]notify.RecordingFileDetail, 0, len(files)) + for _, f := range files { + if f.Size > 0 { + details = append(details, notify.RecordingFileDetail{ + Name: filepath.Base(f.Path), + Size: f.Size, + }) + } + } + return details +} + // EnqueueTask 添加任务到队列 func (m *Manager) EnqueueTask(task *PipelineTask) error { if err := m.store.CreateTask(m.ctx, task); err != nil { diff --git a/src/pipeline/types.go b/src/pipeline/types.go index b2a3e3e..2d76438 100644 --- a/src/pipeline/types.go +++ b/src/pipeline/types.go @@ -4,6 +4,7 @@ package pipeline import ( "context" + "os" "time" "github.com/bililive-go/bililive-go/src/live" @@ -30,18 +31,23 @@ const ( // FileInfo 表示管道中流转的文件信息 type FileInfo struct { Path string `json:"path"` // 文件绝对路径 + Size int64 `json:"size,omitempty"` // 文件大小（字节），在任务创建时记录 Type FileType `json:"type"` // 文件类型 SourcePath string `json:"source_path,omitempty"` // 来源文件路径（用于追踪转换链） Metadata map[string]any `json:"metadata,omitempty"` // 额外元数据 Deletable bool `json:"deletable,omitempty"` // 标记此文件在管道全部成功后可删除 } -// NewVideoFileInfo 创建视频文件信息 +// NewVideoFileInfo 创建视频文件信息，自动记录文件大小 func NewVideoFileInfo(path string) FileInfo { - return FileInfo{ + fi := FileInfo{ Path: path, Type: FileTypeVideo, } + if stat, err := os.Stat(path); err == nil { + fi.Size = stat.Size() + } + return fi } // NewCoverFileInfo 创建封面文件信息 diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index dfc9753..d1a5b1f 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -281,6 +281,10 @@ type recorder struct { currentFileLock sync.RWMutex currentFilePath string + // pipelineEnqueued 标记 Pipeline 任务已入队，由 Pipeline Manager 负责发送录制摘要 + // 避免 recorder 在 Pipeline 处理完成前发送不准确的原始文件列表 + pipelineEnqueued bool + // 当前录制的流信息（来自平台 API） currentStreamInfo *live.AvailableStreamInfo @@ -787,6 +791,7 @@ func (r *recorder) tryRecord(ctx context.Context) { if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles); err != nil { r.getLogger().WithError(err).Error("failed to enqueue pipeline task") } else { + r.pipelineEnqueued = true // 记录 Pipeline 阶段顺序 var stageNames []string for _, stage := range pipelineConfig.Stages { @@ -960,6 +965,12 @@ func (r *recorder) accumulateRecordedFiles(files ...string) { func (r *recorder) sendAccumulatedSummary() { r.recordedFilesMu.Lock() defer r.recordedFilesMu.Unlock() + // Pipeline 已入队，由 Pipeline Manager 在任务完成后发送摘要， + // 确保通知反映 Pipeline 处理后的最终文件状态（而非原始录制文件） + if r.pipelineEnqueued { + r.getLogger().Infof("Pipeline 任务已入队，跳过录制摘要推送（由 Pipeline Manager 负责）") + return + } if r.suppressSummary { r.getLogger().Infof("录制摘要推送已抑制（分段重启），累积 %d 个文件将传递给新 recorder", len(r.recordedFiles)) return From ce35cfd26ce22b0fd0ffb01582589f06ecc6c212 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:13:09 +0800 Subject: [PATCH 24/52] =?UTF-8?q?fix(notify):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=BD=95=E5=88=B6=E6=91=98=E8=A6=81=E9=80=9A=E7=9F=A5=E5=9C=A8?= =?UTF-8?q?=20Pipeline=20=E5=A4=84=E7=90=86=E5=89=8D=E5=8F=91=E9=80=81?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=96=87=E4=BB=B6=E5=88=97=E8=A1=A8=E4=B8=8D?= =?UTF-8?q?=E5=87=86=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 录制摘要通知原来在 recorder.run() 退出时立即发送，此时 Pipeline（转码、修复、上传等） 尚未执行。开启「上传后删除」时，通知中的文件实际已被删除，消息严重失真。 修复方案： - recorder 中添加 pipelineEnqueued 标志，Pipeline 任务入队后跳过摘要推送 - Pipeline Manager 在任务完成/失败后发送摘要，反映最终文件状态 - FileInfo 添加 Size 字段，创建时预存文件大小，确保文件删除后仍可构建摘要 - 提取 sendToAllChannels 消除通道推送重复代码 - 新增 buildUploadedSummaryBody 构造「已上传到云端」场景的消息体 Co-Authored-By: Kscc --- src/notify/send.go | 76 +++++++++++++++++++- src/pipeline/executor.go | 17 +++++ src/pipeline/manager.go | 30 +++++++- src/pipeline/types.go | 19 ++++- src/recorders/recorder.go | 144 +++++++++++++++++++++++++++++++++++--- 5 files changed, 271 insertions(+), 15 deletions(-) diff --git a/src/notify/send.go b/src/notify/send.go index 14db3c4..da8a647 100644 --- a/src/notify/send.go +++ b/src/notify/send.go @@ -16,8 +16,9 @@ import ( // RecordingFileDetail 录制文件详情 type RecordingFileDetail struct { - Name string // 文件名（不含路径） - Size int64 // 文件大小（字节） + Name string // 文件名（不含路径） + Size int64 // 文件大小（字节） + Uploaded bool // 是否已上传到云端（用于摘要消息标注） } // SendNotification 发送统一通知函数 @@ -250,6 +251,29 @@ func buildRecordingSummaryMessage(hostName, platform string, files []RecordingFi return } +// buildUploadedSummaryBody 构造"已上传到云端"场景的录制摘要消息体 +// 与 buildRecordingSummaryMessage 类似，但标注文件已上传、不显示磁盘空间 +func buildUploadedSummaryBody(platform string, files []RecordingFileDetail) string { + const maxDisplayFiles = 30 + + var sb strings.Builder + fmt.Fprintf(&sb, "平台：%s\n", platform) + fmt.Fprintf(&sb, "录制文件：%d 个（已上传到云端）\n", len(files)) + var totalSize int64 + for i, f := range files { + totalSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(files) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(files)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "总大小：%s\n", formatFileSize(totalSize)) + fmt.Fprintf(&sb, "本地文件已清理") + return sb.String() +} + // SendRecordingSummary 录制结束后发送录制文件摘要通知 // outputPath 为录制输出路径，用于获取剩余磁盘空间 func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform string, files []RecordingFileDetail, outputPath string) { @@ -262,7 +286,55 @@ func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform stri } title, body := buildRecordingSummaryMessage(hostName, platform, files, outputPath) + sendToAllChannels(cfg, logger, title, body) +} +// SendPipelineRecordingSummary Pipeline 完成后发送录制摘要通知 +// originalFiles: 原始录制文件列表（allUploaded 时用于显示） +// finalFiles: Pipeline 处理后的最终文件列表（含 Uploaded 标记） +func SendPipelineRecordingSummary( + logger *livelogger.LiveLogger, + hostName, platform string, + originalFiles, finalFiles []RecordingFileDetail, + outputPath string, +) { + cfg := configs.GetCurrentConfig() + if cfg == nil || !cfg.Notify.SendRecordingSummary { + return + } + + // 分离已上传文件和保留文件 + var kept []RecordingFileDetail + allUploaded := len(finalFiles) > 0 + for _, f := range finalFiles { + if f.Uploaded { + // 已上传文件不显示在最终列表中（已从磁盘删除） + } else { + allUploaded = false + kept = append(kept, f) + } + } + + var title, body string + if allUploaded { + // 所有文件已上传并删除：显示原始文件列表 + 上传状态 + if len(originalFiles) == 0 { + return + } + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildUploadedSummaryBody(platform, originalFiles) + } else if len(kept) > 0 { + // 有文件保留在磁盘：显示保留的文件列表 + title, body = buildRecordingSummaryMessage(hostName, platform, kept, outputPath) + } else { + return + } + + sendToAllChannels(cfg, logger, title, body) +} + +// sendToAllChannels 向所有已启用的通知通道推送摘要消息 +func sendToAllChannels(cfg *configs.Config, logger *livelogger.LiveLogger, title, body string) { // Telegram if cfg.Notify.Telegram.Enable { msg := fmt.Sprintf("%s\n%s", title, body) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 02323be..793c04d 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -114,6 +114,9 @@ func (e *Executor) Execute( output, commands, logs, err = e.executeStage(ctx, stageCfg, files) } + // 回填新产出文件的 Size（阶段内构造 FileInfo 时可能未 stat） + backfillFileSizes(output) + // 记录结果 result := StageResult{ StageName: stageCfg.Name, @@ -173,6 +176,8 @@ func (e *Executor) Execute( } return results, ctx.Ctx.Err() } + // 保存最后阶段输出的快照（用于回调提取上传文件详情），再执行清理 + ctx.LastStageFiles = files keptFiles := e.deleteMarkedFiles(files) if len(results) > 0 { results[len(results)-1].OutputFiles = keptFiles @@ -472,6 +477,18 @@ func deduplicateFiles(files []FileInfo) []FileInfo { return result } +// backfillFileSizes 对 Size 为 0 的文件执行 os.Stat 回填大小 +// 阶段内构造 FileInfo 时可能未 stat，导致摘要通知漏掉这些文件 +func backfillFileSizes(files []FileInfo) { + for i := range files { + if files[i].Size == 0 { + if fi, err := os.Stat(files[i].Path); err == nil { + files[i].Size = fi.Size() + } + } + } +} + // getTimeNow 获取当前时间（可用于测试时 mock） var getTimeNow = func() time.Time { return time.Now() diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 1f7fab1..13683f5 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -13,6 +13,7 @@ import ( "github.com/bililive-go/bililive-go/src/configs" "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" + "github.com/bililive-go/bililive-go/src/notify" "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry" @@ -270,6 +271,9 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { logrus.WithField("task_id", task.ID).Info("pipeline task completed successfully") } + // 保存最后阶段输出的快照（清理前），用于回调提取上传文件详情 + task.LastStageFiles = pipelineCtx.LastStageFiles + // 更新任务状态 if err := m.store.UpdateTask(m.ctx, task); err != nil { logrus.WithError(err).Error("failed to update pipeline task status after execution") @@ -277,6 +281,11 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { // 广播任务状态变化 m.broadcastTaskUpdate(task) + + // 调用任务完成回调（recorder 用于追踪 Pipeline 状态并统一发送摘要） + if task.OnTaskComplete != nil { + task.OnTaskComplete(task) + } } // broadcastTaskUpdate 广播任务更新事件 @@ -286,6 +295,22 @@ func (m *Manager) broadcastTaskUpdate(task *PipelineTask) { } } +// FileInfoToDetails 将 FileInfo 切片转换为 RecordingFileDetail 切片 +// 使用 FileInfo 中预存的 Size，无需再次 stat 文件 +// 仅包含视频和其他类型文件，排除封面（封面不在录制摘要中展示） +func FileInfoToDetails(files []FileInfo) []notify.RecordingFileDetail { + details := make([]notify.RecordingFileDetail, 0, len(files)) + for _, f := range files { + if f.Size > 0 && f.Type != FileTypeCover { + details = append(details, notify.RecordingFileDetail{ + Name: filepath.Base(f.Path), + Size: f.Size, + }) + } + } + return details +} + // EnqueueTask 添加任务到队列 func (m *Manager) EnqueueTask(task *PipelineTask) error { if err := m.store.CreateTask(m.ctx, task); err != nil { @@ -309,10 +334,12 @@ func (m *Manager) EnqueueTask(task *PipelineTask) error { // EnqueueRecordingTask 创建并入队录制完成后的处理任务 // 这是 recorder.go 调用的主要入口 +// onTaskComplete: 任务完成回调（可选），用于通知 recorder 任务结束 func (m *Manager) EnqueueRecordingTask( info *live.Info, pipelineConfig *PipelineConfig, outputFiles []string, + onTaskComplete func(task *PipelineTask), ) error { // 构建文件信息列表 files := make([]FileInfo, len(outputFiles)) @@ -322,6 +349,7 @@ func (m *Manager) EnqueueRecordingTask( // 创建任务 task := NewPipelineTask(NewRecordInfo(info), pipelineConfig, files) + task.OnTaskComplete = onTaskComplete return m.EnqueueTask(task) } @@ -644,7 +672,7 @@ func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped [] platform, hostName := inferUploadPathInfo(relPath) recordInfo := RecordInfo{ - LiveID: types.LiveID("manual-upload"), + LiveID: types.LiveID(ManualUploadLiveID), Platform: platform, // 用于上传路径模板：{{ .Platform }} HostName: hostName, // 用于上传路径模板：{{ .HostName }} RoomName: "", // 手动上传无房间名，不影响路径模板 diff --git a/src/pipeline/types.go b/src/pipeline/types.go index b2a3e3e..97d9c59 100644 --- a/src/pipeline/types.go +++ b/src/pipeline/types.go @@ -4,6 +4,7 @@ package pipeline import ( "context" + "os" "time" "github.com/bililive-go/bililive-go/src/live" @@ -15,6 +16,9 @@ import ( // PipelineTaskUpdateEvent 管道任务更新事件 const PipelineTaskUpdateEvent events.EventType = "PipelineTaskUpdate" +// ManualUploadLiveID 手动上传任务的 LiveID 标识 +const ManualUploadLiveID = "manual-upload" + // FileType 文件类型 type FileType string @@ -30,18 +34,23 @@ const ( // FileInfo 表示管道中流转的文件信息 type FileInfo struct { Path string `json:"path"` // 文件绝对路径 + Size int64 `json:"size,omitempty"` // 文件大小（字节），在任务创建时记录 Type FileType `json:"type"` // 文件类型 SourcePath string `json:"source_path,omitempty"` // 来源文件路径（用于追踪转换链） Metadata map[string]any `json:"metadata,omitempty"` // 额外元数据 Deletable bool `json:"deletable,omitempty"` // 标记此文件在管道全部成功后可删除 } -// NewVideoFileInfo 创建视频文件信息 +// NewVideoFileInfo 创建视频文件信息，自动记录文件大小 func NewVideoFileInfo(path string) FileInfo { - return FileInfo{ + fi := FileInfo{ Path: path, Type: FileTypeVideo, } + if stat, err := os.Stat(path); err == nil { + fi.Size = stat.Size() + } + return fi } // NewCoverFileInfo 创建封面文件信息 @@ -84,6 +93,10 @@ type PipelineContext struct { // FFmpegPath 是 ffmpeg 可执行文件的路径 FFmpegPath string + + // LastStageFiles 最后阶段输出的文件快照（清理前保存） + // 用于回调提取上传文件详情，因为 deleteMarkedFiles 会删除文件 + LastStageFiles []FileInfo } // Stage 管道阶段接口 @@ -243,6 +256,8 @@ type PipelineTask struct { CompletedAt *time.Time `json:"completed_at,omitempty"` ErrorMessage string `json:"error_message,omitempty"` CanRetry bool `json:"can_retry"` // 是否可以重试 + OnTaskComplete func(task *PipelineTask) `json:"-"` // 任务完成回调（不持久化） + LastStageFiles []FileInfo `json:"-"` // 最后阶段输出文件（清理前快照，不持久化） } // NewPipelineTask 创建新的管道任务 diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index dfc9753..e1f7fb3 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -281,6 +281,18 @@ type recorder struct { currentFileLock sync.RWMutex currentFilePath string + // pipelineEnqueued 标记 Pipeline 任务已入队，由 Pipeline Manager 负责发送录制摘要 + // 避免 recorder 在 Pipeline 处理完成前发送不准确的原始文件列表 + pipelineEnqueued bool + // pipelinePendingCount 追踪尚未完成的 Pipeline 任务数量 + // 仅当所有任务完成且录制结束后，才发送统一的录制摘要 + pipelinePendingCount int + // pipelineMu 保护 pipelinePendingCount 和 pipelineDetails 的并发访问 + pipelineMu sync.Mutex + // pipelineDetails 收集各 Pipeline 任务的最终文件详情 + // 任务完成时通过回调写入，录制结束时读取用于构造摘要 + pipelineDetails []notify.RecordingFileDetail + // 当前录制的流信息（来自平台 API） currentStreamInfo *live.AvailableStreamInfo @@ -784,9 +796,11 @@ func (r *recorder) tryRecord(ctx context.Context) { } // 入队 Pipeline 任务 - if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles); err != nil { + if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles, r.onPipelineTaskComplete); err != nil { r.getLogger().WithError(err).Error("failed to enqueue pipeline task") } else { + r.pipelineEnqueued = true + r.pipelinePendingCount++ // 记录 Pipeline 阶段顺序 var stageNames []string for _, stage := range pipelineConfig.Stages { @@ -958,22 +972,26 @@ func (r *recorder) accumulateRecordedFiles(files ...string) { // sendAccumulatedSummary 录制结束后统一推送录制文件摘要通知 // 在 run() 退出时通过 defer 调用，确保所有分段文件汇总为一条通知 func (r *recorder) sendAccumulatedSummary() { + r.pipelineMu.Lock() + pendingCount := r.pipelinePendingCount + r.pipelineMu.Unlock() + + // Pipeline 任务尚未全部完成，等待回调触发最终摘要 + if r.pipelineEnqueued && pendingCount > 0 { + r.getLogger().Infof("Pipeline 任务尚未全部完成（剩余 %d 个），延迟摘要推送", pendingCount) + return + } + r.recordedFilesMu.Lock() defer r.recordedFilesMu.Unlock() if r.suppressSummary { r.getLogger().Infof("录制摘要推送已抑制（分段重启），累积 %d 个文件将传递给新 recorder", len(r.recordedFiles)) return } - if len(r.recordedFiles) == 0 { + if len(r.recordedFiles) == 0 && len(r.pipelineDetails) == 0 { r.getLogger().Info("无录制文件，跳过摘要推送") return } - obj, err := r.cache.Get(r.Live) - if err != nil { - r.getLogger().WithError(err).Error("获取直播信息失败，无法推送录制摘要") - return - } - info := obj.(*live.Info) // 获取录制输出路径，用于查询剩余磁盘空间 cfg := configs.GetCurrentConfig() @@ -984,8 +1002,114 @@ func (r *recorder) sendAccumulatedSummary() { outputPath = resolved.OutPutPath } - r.getLogger().Infof("推送录制摘要：%d 个文件", len(r.recordedFiles)) - notify.SendRecordingSummary(r.getLogger(), info.HostName, r.Live.GetPlatformCNName(), r.recordedFiles, outputPath) + // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 + if len(r.pipelineDetails) > 0 { + r.getLogger().Infof("推送 Pipeline 录制摘要：%d 个文件", len(r.pipelineDetails)) + hostName := "" + platform := r.Live.GetPlatformCNName() + if obj, cacheErr := r.cache.Get(r.Live); cacheErr == nil { + if info, ok := obj.(*live.Info); ok { + hostName = info.HostName + } + } + notify.SendPipelineRecordingSummary( + r.getLogger(), + hostName, + platform, + r.recordedFiles, // originalFiles（allUploaded 时显示用） + r.pipelineDetails, // finalFiles + outputPath, + ) + } else { + // Pipeline 未产出有效文件详情，回退到录制累积 + if len(r.recordedFiles) == 0 { + return + } + obj, err := r.cache.Get(r.Live) + if err != nil { + r.getLogger().WithError(err).Error("获取直播信息失败，无法推送录制摘要") + return + } + info := obj.(*live.Info) + r.getLogger().Infof("推送录制摘要：%d 个文件", len(r.recordedFiles)) + notify.SendRecordingSummary(r.getLogger(), info.HostName, r.Live.GetPlatformCNName(), r.recordedFiles, outputPath) + } +} + +// onPipelineTaskComplete Pipeline 任务完成回调 +// 由 Pipeline Manager 在 executeTask 结束时调用 +func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { + r.pipelineMu.Lock() + r.pipelinePendingCount-- + + // 收集任务的最终文件详情 + switch task.Status { + case pipeline.PipelineStatusCompleted: + finalFiles := task.CurrentFiles + if len(finalFiles) == 0 { + // 所有文件已上传并删除：从 LastStageFiles 提取真正上传的文件 + uploadedDetails := extractUploadedDetails(task.LastStageFiles) + if len(uploadedDetails) > 0 { + r.pipelineDetails = append(r.pipelineDetails, uploadedDetails...) + } else { + // 回退：无上传标记时用原始文件 + for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { + d.Uploaded = true + r.pipelineDetails = append(r.pipelineDetails, d) + } + } + } else { + r.pipelineDetails = append(r.pipelineDetails, pipeline.FileInfoToDetails(finalFiles)...) + } + case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled: + // 失败/取消时优先用 CurrentFiles（最后成功阶段的输出），回退到 InitialFiles + if len(task.CurrentFiles) > 0 { + r.pipelineDetails = append(r.pipelineDetails, pipeline.FileInfoToDetails(task.CurrentFiles)...) + } else { + r.pipelineDetails = append(r.pipelineDetails, pipeline.FileInfoToDetails(task.InitialFiles)...) + } + } + + remaining := r.pipelinePendingCount + + // 防止回调被重复执行（如内存中重试场景） + task.OnTaskComplete = nil + + r.pipelineMu.Unlock() + + r.getLogger().Infof("Pipeline 任务完成（状态: %s），剩余 %d 个任务", task.Status, remaining) + + // 所有任务完成且录制已结束，发送统一摘要 + // 使用 goroutine 避免通知发送（Telegram/Email 等）阻塞 Pipeline 任务槽位 + if remaining == 0 { + select { + case <-r.done: + // run() 已退出，异步发送摘要（不阻塞 executeTask） + go r.sendAccumulatedSummary() + default: + // run() 仍在运行，等待 run() 退出时由 sendAccumulatedSummary 发送 + } + } +} + +// extractUploadedDetails 从最后阶段输出文件中提取有上传标记的文件详情 +// CloudUploadStage 会为成功上传的文件设置 Metadata["uploaded"]=true +func extractUploadedDetails(files []pipeline.FileInfo) []notify.RecordingFileDetail { + var details []notify.RecordingFileDetail + for _, f := range files { + if f.Metadata != nil { + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + if f.Size > 0 { + details = append(details, notify.RecordingFileDetail{ + Name: filepath.Base(f.Path), + Size: f.Size, + Uploaded: true, + }) + } + } + } + } + return details } func (r *recorder) logStreamURLRetry(err error) { From ae45e1235fcf57c2367cd12fe9435b480283f0ad Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:25:11 +0800 Subject: [PATCH 25/52] =?UTF-8?q?fix(pipeline):=20=E5=9B=9E=E8=B0=83?= =?UTF-8?q?=E6=94=B9=E7=94=A8=20Manager=20=E6=B3=A8=E5=86=8C=E8=A1=A8?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E5=A4=8D=20SQLite=20=E5=8F=8D=E5=BA=8F?= =?UTF-8?q?=E5=88=97=E5=8C=96=E4=B8=A2=E5=A4=B1=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OnTaskComplete 回调原挂在 PipelineTask 对象上，经 SQLiteStore 持久化/反序列化后 字段丢失（json:"-"），导致生产环境下摘要通知彻底静默失效。 修复： - Manager 新增 taskCallbacks map[int64]func(*PipelineTask) 内存注册表 - EnqueueRecordingTask 在 EnqueueTask 成功后按 task.ID 注册回调 - executeTask 按 task.ID 查表调用并清理，不依赖 task 对象字段 - 移除 PipelineTask.OnTaskComplete 字段（不再需要） - sendAccumulatedSummary 合并未入队分段的录制文件 Co-Authored-By: Kscc --- src/pipeline/manager.go | 30 +++++++++++++++++++++++++----- src/pipeline/types.go | 1 - src/recorders/recorder.go | 20 +++++++++++++++----- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 13683f5..8bd6758 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -30,6 +30,7 @@ type Manager struct { executor *Executor config *ManagerConfig runningTasks map[int64]context.CancelFunc + taskCallbacks map[int64]func(*PipelineTask) // 按 task.ID 索引的完成回调（不经过持久化） mu sync.RWMutex wg sync.WaitGroup eventDispatch events.Dispatcher @@ -71,6 +72,7 @@ func NewManager(ctx context.Context, store Store, config *ManagerConfig, dispatc executor: NewExecutor(logrus.StandardLogger()), config: config, runningTasks: make(map[int64]context.CancelFunc), + taskCallbacks: make(map[int64]func(*PipelineTask)), eventDispatch: dispatcher, } @@ -282,9 +284,16 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { // 广播任务状态变化 m.broadcastTaskUpdate(task) - // 调用任务完成回调（recorder 用于追踪 Pipeline 状态并统一发送摘要） - if task.OnTaskComplete != nil { - task.OnTaskComplete(task) + // 调用任务完成回调（从 Manager 注册表查找，避免 SQLite 反序列化丢失） + m.mu.RLock() + callback := m.taskCallbacks[task.ID] + m.mu.RUnlock() + if callback != nil { + callback(task) + // 调用后清理注册表 + m.mu.Lock() + delete(m.taskCallbacks, task.ID) + m.mu.Unlock() } } @@ -349,9 +358,20 @@ func (m *Manager) EnqueueRecordingTask( // 创建任务 task := NewPipelineTask(NewRecordInfo(info), pipelineConfig, files) - task.OnTaskComplete = onTaskComplete - return m.EnqueueTask(task) + if err := m.EnqueueTask(task); err != nil { + return err + } + + // 任务入队成功后注册回调（此时 task.ID 已由 store.AssignID 填充） + // 回调存储在 Manager 内存中，不经过持久化，避免 SQLite 反序列化丢失 + if onTaskComplete != nil { + m.mu.Lock() + m.taskCallbacks[task.ID] = onTaskComplete + m.mu.Unlock() + } + + return nil } // CancelTask 取消任务 diff --git a/src/pipeline/types.go b/src/pipeline/types.go index 97d9c59..28792d9 100644 --- a/src/pipeline/types.go +++ b/src/pipeline/types.go @@ -256,7 +256,6 @@ type PipelineTask struct { CompletedAt *time.Time `json:"completed_at,omitempty"` ErrorMessage string `json:"error_message,omitempty"` CanRetry bool `json:"can_retry"` // 是否可以重试 - OnTaskComplete func(task *PipelineTask) `json:"-"` // 任务完成回调（不持久化） LastStageFiles []FileInfo `json:"-"` // 最后阶段输出文件（清理前快照，不持久化） } diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index e1f7fb3..3525dd2 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -1004,7 +1004,20 @@ func (r *recorder) sendAccumulatedSummary() { // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 if len(r.pipelineDetails) > 0 { - r.getLogger().Infof("推送 Pipeline 录制摘要：%d 个文件", len(r.pipelineDetails)) + // 合并未被 Pipeline 覆盖的录制文件（如某分段因 stages==0 未入队） + merged := r.pipelineDetails + if len(r.recordedFiles) > 0 { + inPipeline := map[string]bool{} + for _, d := range r.pipelineDetails { + inPipeline[d.Name] = true + } + for _, f := range r.recordedFiles { + if !inPipeline[f.Name] { + merged = append(merged, f) + } + } + } + r.getLogger().Infof("推送 Pipeline 录制摘要：%d 个文件", len(merged)) hostName := "" platform := r.Live.GetPlatformCNName() if obj, cacheErr := r.cache.Get(r.Live); cacheErr == nil { @@ -1017,7 +1030,7 @@ func (r *recorder) sendAccumulatedSummary() { hostName, platform, r.recordedFiles, // originalFiles（allUploaded 时显示用） - r.pipelineDetails, // finalFiles + merged, // finalFiles（含未入队分段的文件） outputPath, ) } else { @@ -1072,9 +1085,6 @@ func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { remaining := r.pipelinePendingCount - // 防止回调被重复执行（如内存中重试场景） - task.OnTaskComplete = nil - r.pipelineMu.Unlock() r.getLogger().Infof("Pipeline 任务完成（状态: %s），剩余 %d 个任务", task.Status, remaining) From 0cf8b043329bda75d89a722d27f14e43b18f80c9 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:36:09 +0800 Subject: [PATCH 26/52] =?UTF-8?q?fix(pipeline):=20=E5=88=86=E6=AE=B5?= =?UTF-8?q?=E9=87=8D=E5=90=AF=E6=97=B6=E8=BD=AC=E7=A7=BB=20Pipeline=20?= =?UTF-8?q?=E7=8A=B6=E6=80=81=EF=BC=8C=E9=81=BF=E5=85=8D=E6=97=A7=E5=88=86?= =?UTF-8?q?=E6=AE=B5=E6=91=98=E8=A6=81=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CloseForRestart 只传 recordedFiles，不传 Pipeline 状态。旧回调仍绑定旧 recorder，导致旧分段的 Pipeline 结果（转码/上传后的文件详情）丢失。 修复： - pipelineDetails 改为指针，分段重启时新旧 recorder 共享同一底层数据 - 新增 TransferPipelineState 方法，合并 pendingCount 和 details - RestartRecorder 在文件传递后调用 TransferPipelineState - 旧回调写入 (*old.pipelineDetails) 实际写入新 recorder 能读取的位置 Co-Authored-By: Kscc --- src/notify/send.go | 11 +++- src/pipeline/manager.go | 11 ++++ src/pipeline/stages/extract_cover.go | 4 ++ src/recorders/manager.go | 7 +++ src/recorders/recorder.go | 85 +++++++++++++++++++++------- 5 files changed, 94 insertions(+), 24 deletions(-) diff --git a/src/notify/send.go b/src/notify/send.go index da8a647..0539667 100644 --- a/src/notify/send.go +++ b/src/notify/send.go @@ -317,12 +317,17 @@ func SendPipelineRecordingSummary( var title, body string if allUploaded { - // 所有文件已上传并删除：显示原始文件列表 + 上传状态 - if len(originalFiles) == 0 { + // 所有文件已上传并删除：显示实际上传的文件列表 + displayFiles := finalFiles + if len(displayFiles) == 0 { + // 回退：finalFiles 为空时用 originalFiles + displayFiles = originalFiles + } + if len(displayFiles) == 0 { return } title = fmt.Sprintf("%s 录制完成", hostName) - body = buildUploadedSummaryBody(platform, originalFiles) + body = buildUploadedSummaryBody(platform, displayFiles) } else if len(kept) > 0 { // 有文件保留在磁盘：显示保留的文件列表 title, body = buildRecordingSummaryMessage(hostName, platform, kept, outputPath) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 8bd6758..845933b 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -396,6 +396,17 @@ func (m *Manager) CancelTask(taskID int64) error { return err } m.broadcastTaskUpdate(task) + + // 触发完成回调（recorder 需要递减 pipelinePendingCount） + m.mu.RLock() + callback := m.taskCallbacks[taskID] + m.mu.RUnlock() + if callback != nil { + callback(task) + m.mu.Lock() + delete(m.taskCallbacks, taskID) + m.mu.Unlock() + } } return nil diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index 221fe38..ee4f90d 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -334,6 +334,10 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli output[i].Metadata = map[string]any{} } output[i].Metadata["delete_all"] = true + // 为实际上传的文件标记 uploaded（供回调提取上传文件详情） + if uploadedIndices[i] { + output[i].Metadata["uploaded"] = true + } } s.logs += fmt.Sprintf("全部上传成功，已标记 %d 个文件待删除（含中间产物）\n", len(output)) ctx.Logger.Infof("全部上传成功，已标记 %d 个文件待删除（含中间产物）", len(output)) diff --git a/src/recorders/manager.go b/src/recorders/manager.go index 6925772..3e43a5b 100644 --- a/src/recorders/manager.go +++ b/src/recorders/manager.go @@ -274,6 +274,13 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error { } } + // 4. 转移 Pipeline 状态（确保旧分段的 Pipeline 结果不丢失） + if newRecTyped, ok := newRec.(*recorder); ok { + if oldRecTyped, ok := oldRecorder.(*recorder); ok { + newRecTyped.TransferPipelineState(oldRecTyped) + } + } + return nil } diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index 3525dd2..c967a20 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -291,7 +291,10 @@ type recorder struct { pipelineMu sync.Mutex // pipelineDetails 收集各 Pipeline 任务的最终文件详情 // 任务完成时通过回调写入，录制结束时读取用于构造摘要 - pipelineDetails []notify.RecordingFileDetail + // 使用指针，分段重启时新旧 recorder 共享同一底层数据 + pipelineDetails *[]notify.RecordingFileDetail + // summarySent 标记摘要已发送，防止 defer 和回调并发双发 + summarySent bool // 当前录制的流信息（来自平台 API） currentStreamInfo *live.AvailableStreamInfo @@ -325,14 +328,15 @@ func NewRecorder(ctx context.Context, live live.Live) (Recorder, error) { inst := instance.GetInstance(ctx) return &recorder{ - Live: live, - cache: inst.Cache, - startTime: time.Now(), - ed: inst.EventDispatcher.(events.Dispatcher), - state: begin, - stop: make(chan struct{}), - done: make(chan struct{}), - parserLock: new(sync.RWMutex), + Live: live, + cache: inst.Cache, + startTime: time.Now(), + ed: inst.EventDispatcher.(events.Dispatcher), + state: begin, + stop: make(chan struct{}), + done: make(chan struct{}), + parserLock: new(sync.RWMutex), + pipelineDetails: new([]notify.RecordingFileDetail), }, nil } @@ -795,12 +799,20 @@ func (r *recorder) tryRecord(ctx context.Context) { return } + // 先递增 pending 计数（在入队前，避免任务被异步调度完成后回调递减到 -1） + r.pipelineMu.Lock() + r.pipelinePendingCount++ + r.pipelineMu.Unlock() + // 入队 Pipeline 任务 if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles, r.onPipelineTaskComplete); err != nil { r.getLogger().WithError(err).Error("failed to enqueue pipeline task") + // 入队失败，回滚计数 + r.pipelineMu.Lock() + r.pipelinePendingCount-- + r.pipelineMu.Unlock() } else { r.pipelineEnqueued = true - r.pipelinePendingCount++ // 记录 Pipeline 阶段顺序 var stageNames []string for _, stage := range pipelineConfig.Stages { @@ -974,13 +986,20 @@ func (r *recorder) accumulateRecordedFiles(files ...string) { func (r *recorder) sendAccumulatedSummary() { r.pipelineMu.Lock() pendingCount := r.pipelinePendingCount - r.pipelineMu.Unlock() - + // 幂等保护：防止 defer 和回调并发双发 + if r.summarySent { + r.pipelineMu.Unlock() + return + } // Pipeline 任务尚未全部完成，等待回调触发最终摘要 if r.pipelineEnqueued && pendingCount > 0 { + r.pipelineMu.Unlock() r.getLogger().Infof("Pipeline 任务尚未全部完成（剩余 %d 个），延迟摘要推送", pendingCount) return } + // 标记摘要已发送（在实际发送前，防止并发双发） + r.summarySent = true + r.pipelineMu.Unlock() r.recordedFilesMu.Lock() defer r.recordedFilesMu.Unlock() @@ -988,7 +1007,7 @@ func (r *recorder) sendAccumulatedSummary() { r.getLogger().Infof("录制摘要推送已抑制（分段重启），累积 %d 个文件将传递给新 recorder", len(r.recordedFiles)) return } - if len(r.recordedFiles) == 0 && len(r.pipelineDetails) == 0 { + if len(r.recordedFiles) == 0 && len(*r.pipelineDetails) == 0 { r.getLogger().Info("无录制文件，跳过摘要推送") return } @@ -1003,12 +1022,12 @@ func (r *recorder) sendAccumulatedSummary() { } // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 - if len(r.pipelineDetails) > 0 { + if len((*r.pipelineDetails)) > 0 { // 合并未被 Pipeline 覆盖的录制文件（如某分段因 stages==0 未入队） - merged := r.pipelineDetails + merged := (*r.pipelineDetails) if len(r.recordedFiles) > 0 { inPipeline := map[string]bool{} - for _, d := range r.pipelineDetails { + for _, d := range (*r.pipelineDetails) { inPipeline[d.Name] = true } for _, f := range r.recordedFiles { @@ -1063,23 +1082,23 @@ func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { // 所有文件已上传并删除：从 LastStageFiles 提取真正上传的文件 uploadedDetails := extractUploadedDetails(task.LastStageFiles) if len(uploadedDetails) > 0 { - r.pipelineDetails = append(r.pipelineDetails, uploadedDetails...) + (*r.pipelineDetails) = append((*r.pipelineDetails), uploadedDetails...) } else { // 回退：无上传标记时用原始文件 for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { d.Uploaded = true - r.pipelineDetails = append(r.pipelineDetails, d) + (*r.pipelineDetails) = append((*r.pipelineDetails), d) } } } else { - r.pipelineDetails = append(r.pipelineDetails, pipeline.FileInfoToDetails(finalFiles)...) + (*r.pipelineDetails) = append((*r.pipelineDetails), pipeline.FileInfoToDetails(finalFiles)...) } case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled: // 失败/取消时优先用 CurrentFiles（最后成功阶段的输出），回退到 InitialFiles if len(task.CurrentFiles) > 0 { - r.pipelineDetails = append(r.pipelineDetails, pipeline.FileInfoToDetails(task.CurrentFiles)...) + (*r.pipelineDetails) = append((*r.pipelineDetails), pipeline.FileInfoToDetails(task.CurrentFiles)...) } else { - r.pipelineDetails = append(r.pipelineDetails, pipeline.FileInfoToDetails(task.InitialFiles)...) + (*r.pipelineDetails) = append((*r.pipelineDetails), pipeline.FileInfoToDetails(task.InitialFiles)...) } } @@ -1277,6 +1296,30 @@ func (r *recorder) SetInitialRecordedFiles(files []notify.RecordingFileDetail) { r.getLogger().Infof("继承上一个 recorder 的 %d 个录制文件", len(files)) } +// TransferPipelineState 从旧 recorder 转移 Pipeline 状态 +// 分段重启时由 RestartRecorder 调用，确保旧分段的 Pipeline 结果不丢失 +func (r *recorder) TransferPipelineState(old *recorder) { + r.pipelineMu.Lock() + defer r.pipelineMu.Unlock() + + old.pipelineMu.Lock() + defer old.pipelineMu.Unlock() + + // 合并 Pipeline 状态 + r.pipelineEnqueued = r.pipelineEnqueued || old.pipelineEnqueued + r.pipelinePendingCount += old.pipelinePendingCount + + // 合并文件详情（旧 recorder 的详情追加到新 recorder） + *r.pipelineDetails = append(*r.pipelineDetails, *old.pipelineDetails...) + + // 将旧 recorder 的详情指针指向新 recorder 的共享数据 + // 这样旧 recorder 的回调写入时，数据会存到新 recorder 能读取的位置 + old.pipelineDetails = r.pipelineDetails + + r.getLogger().Infof("继承 Pipeline 状态：pending=%d, details=%d 个文件", + old.pipelinePendingCount, len(*old.pipelineDetails)) +} + func (r *recorder) getLogger() *livelogger.LiveLogger { return r.Live.GetLogger() } From 82b63cbdb99ac3fd76b738a46c9a2b48e85e8a3f Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:50:28 +0800 Subject: [PATCH 27/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=88=86=E6=AE=B5=E9=87=8D=E5=90=AF=E6=97=B6=E5=BD=95=E5=88=B6?= =?UTF-8?q?=E6=91=98=E8=A6=81=E4=B8=A2=E5=A4=B1=E5=8F=8A=E5=A4=9A=E5=A4=84?= =?UTF-8?q?=E7=AB=9E=E6=80=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. TransferPipelineState 共享状态指针重定向 - 引入 pipelineSharedState 结构体，将 pendingCount、details、 summarySent 等打包为共享状态 - 分段重启时 old.pipelineState = r.pipelineState 指针重定向， 旧任务回调操作同一份共享计数器，解决 pendingCount 永远无法 归零导致摘要永久延迟的问题 - 新增 onAllTasksDone 回调，旧任务全部完成时触发新 recorder 的摘要发送 2. 摘要合并排除已删除的源文件 - 新增 sourceNames 字段，通过 FileInfo.SourcePath 追踪被 Pipeline 转换的原始文件名 - sendAccumulatedSummary 合并时排除 sourceNames 中的文件， 避免 convert_mp4 delete_source 场景下已删除的 FLV 被重新 纳入摘要 3. 回调注册移入 EnqueueTask 异步调度前 - EnqueueTask 新增 onTaskComplete 参数，在 store.CreateTask 之后、bilisentry.Go(scheduleNextTasks) 之前注册回调 - 消除 executeTask 可能在回调注册前完成的竞态窗口 4. 回调查询-删除原子化 - executeTask 和 CancelTask 中将 RLock 读取 + Lock 删除 合并为单次 Lock 内的读取+删除 - 确保每个任务的回调至多触发一次，避免并发时双发导致 pipelinePendingCount 减为负数 5. runExited 标志替代 r.done 通道检查 - onPipelineTaskComplete 中用 pipelineState.runExited（锁内 读取）替代 select { case <-r.done: } 通道检查 - 消除 sendAccumulatedSummary 和 close(r.done) 之间的 defer 执行顺序导致的摘要漏发窗口 --- src/pipeline/manager.go | 49 ++++------ src/recorders/recorder.go | 167 +++++++++++++++++++++----------- src/servers/pipeline_handler.go | 2 +- 3 files changed, 130 insertions(+), 88 deletions(-) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 845933b..4f99eaa 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -284,16 +284,14 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { // 广播任务状态变化 m.broadcastTaskUpdate(task) - // 调用任务完成回调（从 Manager 注册表查找，避免 SQLite 反序列化丢失） - m.mu.RLock() + // 原子地取出并删除回调，确保每个任务的回调至多触发一次 + // 避免 executeTask 和 CancelTask 并发时双发回调 + m.mu.Lock() callback := m.taskCallbacks[task.ID] - m.mu.RUnlock() + delete(m.taskCallbacks, task.ID) + m.mu.Unlock() if callback != nil { callback(task) - // 调用后清理注册表 - m.mu.Lock() - delete(m.taskCallbacks, task.ID) - m.mu.Unlock() } } @@ -321,11 +319,19 @@ func FileInfoToDetails(files []FileInfo) []notify.RecordingFileDetail { } // EnqueueTask 添加任务到队列 -func (m *Manager) EnqueueTask(task *PipelineTask) error { +// onTaskComplete: 任务完成回调（可选），在异步调度前注册，避免竞态窗口 +func (m *Manager) EnqueueTask(task *PipelineTask, onTaskComplete func(*PipelineTask)) error { if err := m.store.CreateTask(m.ctx, task); err != nil { return fmt.Errorf("failed to create pipeline task: %w", err) } + // 在异步调度前注册回调，确保 executeTask 读取时回调已就绪 + if onTaskComplete != nil { + m.mu.Lock() + m.taskCallbacks[task.ID] = onTaskComplete + m.mu.Unlock() + } + logrus.WithFields(logrus.Fields{ "task_id": task.ID, "initial_files": len(task.InitialFiles), @@ -359,19 +365,8 @@ func (m *Manager) EnqueueRecordingTask( // 创建任务 task := NewPipelineTask(NewRecordInfo(info), pipelineConfig, files) - if err := m.EnqueueTask(task); err != nil { - return err - } - - // 任务入队成功后注册回调（此时 task.ID 已由 store.AssignID 填充） - // 回调存储在 Manager 内存中，不经过持久化，避免 SQLite 反序列化丢失 - if onTaskComplete != nil { - m.mu.Lock() - m.taskCallbacks[task.ID] = onTaskComplete - m.mu.Unlock() - } - - return nil + // 入队并注册回调（EnqueueTask 内部在异步调度前完成注册，避免竞态窗口） + return m.EnqueueTask(task, onTaskComplete) } // CancelTask 取消任务 @@ -397,15 +392,13 @@ func (m *Manager) CancelTask(taskID int64) error { } m.broadcastTaskUpdate(task) - // 触发完成回调（recorder 需要递减 pipelinePendingCount） - m.mu.RLock() + // 原子地取出并删除回调，避免与 executeTask 并发时双发 + m.mu.Lock() callback := m.taskCallbacks[taskID] - m.mu.RUnlock() + delete(m.taskCallbacks, taskID) + m.mu.Unlock() if callback != nil { callback(task) - m.mu.Lock() - delete(m.taskCallbacks, taskID) - m.mu.Unlock() } } @@ -714,7 +707,7 @@ func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped [] files := []FileInfo{NewVideoFileInfo(absPath)} task := NewPipelineTask(recordInfo, uploadConfig, files) - if enqueueErr := m.EnqueueTask(task); enqueueErr != nil { + if enqueueErr := m.EnqueueTask(task, nil); enqueueErr != nil { skipped = append(skipped, filepath.Base(absPath)+" - 入队失败: "+enqueueErr.Error()) continue } diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index c967a20..ee32cae 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -265,6 +265,20 @@ var ( _ danmakuRecorder = (*danmaku.DouyuDanmakuRecorder)(nil) ) +// pipelineSharedState 封装可在分段重启时共享的 Pipeline 状态 +// 分段重启时新旧 recorder 通过指针重定向共享同一份状态， +// 确保旧任务的回调能正确递减新 recorder 的计数器 +type pipelineSharedState struct { + mu sync.Mutex + pendingCount int // 尚未完成的 Pipeline 任务数 + enqueued bool // 是否有 Pipeline 任务已入队 + summarySent bool // 摘要是否已发送（幂等保护） + runExited bool // run() 是否已退出（defer 中置位，回调中检查） + details []notify.RecordingFileDetail // 收集的文件详情 + sourceNames map[string]bool // 被 Pipeline 消费的原始文件名（用于合并时排除已删除的源文件） + onAllTasksDone func() // 所有任务完成时的回调（分段重启时设置） +} + type recorder struct { Live live.Live ed events.Dispatcher @@ -281,20 +295,9 @@ type recorder struct { currentFileLock sync.RWMutex currentFilePath string - // pipelineEnqueued 标记 Pipeline 任务已入队，由 Pipeline Manager 负责发送录制摘要 - // 避免 recorder 在 Pipeline 处理完成前发送不准确的原始文件列表 - pipelineEnqueued bool - // pipelinePendingCount 追踪尚未完成的 Pipeline 任务数量 - // 仅当所有任务完成且录制结束后，才发送统一的录制摘要 - pipelinePendingCount int - // pipelineMu 保护 pipelinePendingCount 和 pipelineDetails 的并发访问 - pipelineMu sync.Mutex - // pipelineDetails 收集各 Pipeline 任务的最终文件详情 - // 任务完成时通过回调写入，录制结束时读取用于构造摘要 - // 使用指针，分段重启时新旧 recorder 共享同一底层数据 - pipelineDetails *[]notify.RecordingFileDetail - // summarySent 标记摘要已发送，防止 defer 和回调并发双发 - summarySent bool + // pipelineState 封装 Pipeline 相关的共享状态 + // 使用指针，分段重启时新旧 recorder 共享同一份状态 + pipelineState *pipelineSharedState // 当前录制的流信息（来自平台 API） currentStreamInfo *live.AvailableStreamInfo @@ -336,7 +339,9 @@ func NewRecorder(ctx context.Context, live live.Live) (Recorder, error) { stop: make(chan struct{}), done: make(chan struct{}), parserLock: new(sync.RWMutex), - pipelineDetails: new([]notify.RecordingFileDetail), + pipelineState: &pipelineSharedState{ + sourceNames: make(map[string]bool), + }, }, nil } @@ -800,19 +805,19 @@ func (r *recorder) tryRecord(ctx context.Context) { } // 先递增 pending 计数（在入队前，避免任务被异步调度完成后回调递减到 -1） - r.pipelineMu.Lock() - r.pipelinePendingCount++ - r.pipelineMu.Unlock() + r.pipelineState.mu.Lock() + r.pipelineState.pendingCount++ + r.pipelineState.mu.Unlock() // 入队 Pipeline 任务 if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles, r.onPipelineTaskComplete); err != nil { r.getLogger().WithError(err).Error("failed to enqueue pipeline task") // 入队失败，回滚计数 - r.pipelineMu.Lock() - r.pipelinePendingCount-- - r.pipelineMu.Unlock() + r.pipelineState.mu.Lock() + r.pipelineState.pendingCount-- + r.pipelineState.mu.Unlock() } else { - r.pipelineEnqueued = true + r.pipelineState.enqueued = true // 记录 Pipeline 阶段顺序 var stageNames []string for _, stage := range pipelineConfig.Stages { @@ -935,7 +940,14 @@ func (r *recorder) selectPreferredStream(streamInfos []*live.StreamUrlInfo) (ret func (r *recorder) run(ctx context.Context) { defer close(r.done) - defer r.sendAccumulatedSummary() + defer func() { + // 标记 run() 已退出，使回调在 remaining==0 时直接发送摘要， + // 而不是等待 r.done 通道关闭（避免 defer 执行顺序导致的漏发窗口） + r.pipelineState.mu.Lock() + r.pipelineState.runExited = true + r.pipelineState.mu.Unlock() + r.sendAccumulatedSummary() + }() const minRetryInterval = 5 * time.Second @@ -984,22 +996,22 @@ func (r *recorder) accumulateRecordedFiles(files ...string) { // sendAccumulatedSummary 录制结束后统一推送录制文件摘要通知 // 在 run() 退出时通过 defer 调用，确保所有分段文件汇总为一条通知 func (r *recorder) sendAccumulatedSummary() { - r.pipelineMu.Lock() - pendingCount := r.pipelinePendingCount + r.pipelineState.mu.Lock() + pendingCount := r.pipelineState.pendingCount // 幂等保护：防止 defer 和回调并发双发 - if r.summarySent { - r.pipelineMu.Unlock() + if r.pipelineState.summarySent { + r.pipelineState.mu.Unlock() return } // Pipeline 任务尚未全部完成，等待回调触发最终摘要 - if r.pipelineEnqueued && pendingCount > 0 { - r.pipelineMu.Unlock() + if r.pipelineState.enqueued && pendingCount > 0 { + r.pipelineState.mu.Unlock() r.getLogger().Infof("Pipeline 任务尚未全部完成（剩余 %d 个），延迟摘要推送", pendingCount) return } // 标记摘要已发送（在实际发送前，防止并发双发） - r.summarySent = true - r.pipelineMu.Unlock() + r.pipelineState.summarySent = true + r.pipelineState.mu.Unlock() r.recordedFilesMu.Lock() defer r.recordedFilesMu.Unlock() @@ -1007,7 +1019,7 @@ func (r *recorder) sendAccumulatedSummary() { r.getLogger().Infof("录制摘要推送已抑制（分段重启），累积 %d 个文件将传递给新 recorder", len(r.recordedFiles)) return } - if len(r.recordedFiles) == 0 && len(*r.pipelineDetails) == 0 { + if len(r.recordedFiles) == 0 && len(r.pipelineState.details) == 0 { r.getLogger().Info("无录制文件，跳过摘要推送") return } @@ -1022,14 +1034,18 @@ func (r *recorder) sendAccumulatedSummary() { } // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 - if len((*r.pipelineDetails)) > 0 { + if len(r.pipelineState.details) > 0 { // 合并未被 Pipeline 覆盖的录制文件（如某分段因 stages==0 未入队） - merged := (*r.pipelineDetails) + merged := r.pipelineState.details if len(r.recordedFiles) > 0 { inPipeline := map[string]bool{} - for _, d := range (*r.pipelineDetails) { + for _, d := range r.pipelineState.details { inPipeline[d.Name] = true } + // 排除已被 Pipeline 转换并删除的源文件（如 convert_mp4 delete_source 场景） + for name := range r.pipelineState.sourceNames { + inPipeline[name] = true + } for _, f := range r.recordedFiles { if !inPipeline[f.Name] { merged = append(merged, f) @@ -1071,8 +1087,8 @@ func (r *recorder) sendAccumulatedSummary() { // onPipelineTaskComplete Pipeline 任务完成回调 // 由 Pipeline Manager 在 executeTask 结束时调用 func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { - r.pipelineMu.Lock() - r.pipelinePendingCount-- + r.pipelineState.mu.Lock() + r.pipelineState.pendingCount-- // 收集任务的最终文件详情 switch task.Status { @@ -1082,42 +1098,50 @@ func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { // 所有文件已上传并删除：从 LastStageFiles 提取真正上传的文件 uploadedDetails := extractUploadedDetails(task.LastStageFiles) if len(uploadedDetails) > 0 { - (*r.pipelineDetails) = append((*r.pipelineDetails), uploadedDetails...) + r.pipelineState.details = append(r.pipelineState.details, uploadedDetails...) } else { // 回退：无上传标记时用原始文件 for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { d.Uploaded = true - (*r.pipelineDetails) = append((*r.pipelineDetails), d) + r.pipelineState.details = append(r.pipelineState.details, d) } } } else { - (*r.pipelineDetails) = append((*r.pipelineDetails), pipeline.FileInfoToDetails(finalFiles)...) + r.pipelineState.details = append(r.pipelineState.details, pipeline.FileInfoToDetails(finalFiles)...) } case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled: // 失败/取消时优先用 CurrentFiles（最后成功阶段的输出），回退到 InitialFiles if len(task.CurrentFiles) > 0 { - (*r.pipelineDetails) = append((*r.pipelineDetails), pipeline.FileInfoToDetails(task.CurrentFiles)...) + r.pipelineState.details = append(r.pipelineState.details, pipeline.FileInfoToDetails(task.CurrentFiles)...) } else { - (*r.pipelineDetails) = append((*r.pipelineDetails), pipeline.FileInfoToDetails(task.InitialFiles)...) + r.pipelineState.details = append(r.pipelineState.details, pipeline.FileInfoToDetails(task.InitialFiles)...) } } - remaining := r.pipelinePendingCount + // 收集被 Pipeline 转换的原始文件名（SourcePath），用于摘要合并时排除已删除的源文件 + collectSourceNames(r.pipelineState.sourceNames, task.CurrentFiles, task.LastStageFiles) - r.pipelineMu.Unlock() + remaining := r.pipelineState.pendingCount + runExited := r.pipelineState.runExited + onAllDone := r.pipelineState.onAllTasksDone + + r.pipelineState.mu.Unlock() r.getLogger().Infof("Pipeline 任务完成（状态: %s），剩余 %d 个任务", task.Status, remaining) // 所有任务完成且录制已结束，发送统一摘要 // 使用 goroutine 避免通知发送（Telegram/Email 等）阻塞 Pipeline 任务槽位 if remaining == 0 { - select { - case <-r.done: + if r.suppressSummary { + // 分段重启场景：旧 recorder 已抑制，通过共享回调触发新 recorder 的摘要 + if onAllDone != nil { + onAllDone() + } + } else if runExited { // run() 已退出，异步发送摘要（不阻塞 executeTask） go r.sendAccumulatedSummary() - default: - // run() 仍在运行，等待 run() 退出时由 sendAccumulatedSummary 发送 } + // run() 仍在运行且未抑制：等待 run() 退出时由 defer 中的 sendAccumulatedSummary 发送 } } @@ -1141,6 +1165,19 @@ func extractUploadedDetails(files []pipeline.FileInfo) []notify.RecordingFileDet return details } +// collectSourceNames 从 Pipeline 输出文件中收集被转换的原始文件名 +// 当 convert_mp4/burn_subtitles 等阶段转换文件时，输出文件的 SourcePath 指向原始文件 +// 这些原始文件可能已被 Pipeline 删除，摘要合并时需要排除，避免将已删除的源文件重新纳入 +func collectSourceNames(sourceNames map[string]bool, fileSets ...[]pipeline.FileInfo) { + for _, files := range fileSets { + for _, f := range files { + if f.SourcePath != "" { + sourceNames[filepath.Base(f.SourcePath)] = true + } + } + } +} + func (r *recorder) logStreamURLRetry(err error) { if configs.IsDebug() || !strings.EqualFold(r.Live.GetPlatformCNName(), "SOOP") { r.warnStreamURLRetry(err, "failed to get stream url, will retry after 5s...", 0) @@ -1299,25 +1336,37 @@ func (r *recorder) SetInitialRecordedFiles(files []notify.RecordingFileDetail) { // TransferPipelineState 从旧 recorder 转移 Pipeline 状态 // 分段重启时由 RestartRecorder 调用，确保旧分段的 Pipeline 结果不丢失 func (r *recorder) TransferPipelineState(old *recorder) { - r.pipelineMu.Lock() - defer r.pipelineMu.Unlock() + r.pipelineState.mu.Lock() + defer r.pipelineState.mu.Unlock() - old.pipelineMu.Lock() - defer old.pipelineMu.Unlock() + old.pipelineState.mu.Lock() + defer old.pipelineState.mu.Unlock() // 合并 Pipeline 状态 - r.pipelineEnqueued = r.pipelineEnqueued || old.pipelineEnqueued - r.pipelinePendingCount += old.pipelinePendingCount + r.pipelineState.enqueued = r.pipelineState.enqueued || old.pipelineState.enqueued + r.pipelineState.pendingCount += old.pipelineState.pendingCount // 合并文件详情（旧 recorder 的详情追加到新 recorder） - *r.pipelineDetails = append(*r.pipelineDetails, *old.pipelineDetails...) + r.pipelineState.details = append(r.pipelineState.details, old.pipelineState.details...) - // 将旧 recorder 的详情指针指向新 recorder 的共享数据 - // 这样旧 recorder 的回调写入时，数据会存到新 recorder 能读取的位置 - old.pipelineDetails = r.pipelineDetails + // 合并已被 Pipeline 消费的原始文件名 + for name := range old.pipelineState.sourceNames { + r.pipelineState.sourceNames[name] = true + } + + // 关键：将旧 recorder 的 pipelineState 指针重定向到新 recorder 的共享状态 + // 这样旧任务完成时的回调会操作新 recorder 的计数器 + old.pipelineState = r.pipelineState + + // 设置回调：旧任务全部完成时，由旧回调触发新 recorder 的摘要发送 + if old.suppressSummary { + r.pipelineState.onAllTasksDone = func() { + go r.sendAccumulatedSummary() + } + } r.getLogger().Infof("继承 Pipeline 状态：pending=%d, details=%d 个文件", - old.pipelinePendingCount, len(*old.pipelineDetails)) + r.pipelineState.pendingCount, len(r.pipelineState.details)) } func (r *recorder) getLogger() *livelogger.LiveLogger { diff --git a/src/servers/pipeline_handler.go b/src/servers/pipeline_handler.go index f01676c..d084c2d 100644 --- a/src/servers/pipeline_handler.go +++ b/src/servers/pipeline_handler.go @@ -347,7 +347,7 @@ func makeBatchBurnHandler(pm *pipeline.Manager) http.HandlerFunc { } task := pipeline.NewPipelineTask(recordInfo, burnConfig, files) - if err := pm.EnqueueTask(task); err != nil { + if err := pm.EnqueueTask(task, nil); err != nil { result.Skipped = append(result.Skipped, filepath.Base(p)+" - 入队失败: "+err.Error()) continue } From 04ffe41c3c40bdbd65b7e6abcbeb323b521d95b4 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:09:10 +0800 Subject: [PATCH 28/52] =?UTF-8?q?fix(pipeline):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=88=86=E6=AE=B5=E9=87=8D=E5=90=AF=E6=97=B6=20Pipeline=20?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E8=BD=AC=E7=A7=BB=E7=9A=84=E4=B8=89=E4=B8=AA?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 连续重启 A→B→C 回调链断裂：用 redirectedTo 链替代指针覆盖， resolveState() 沿链找到最终活跃状态，回调正确操作最终 recorder 的计数器 2. data race：suppressSummary 从 recorder 移入 pipelineSharedState 用 mu 保护； r.pipelineState 指针字段不再被覆盖，消除 Lock/Unlock 不配对的竞态 3. extractUploadedDetails 未排除封面文件：增加 FileTypeCover 过滤， 与 FileInfoToDetails 保持一致 Co-Authored-By: Kscc --- src/recorders/recorder.go | 132 ++++++++----- src/recorders/recorder_test.go | 333 +++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+), 44 deletions(-) diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index ee32cae..babafe4 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -266,7 +266,7 @@ var ( ) // pipelineSharedState 封装可在分段重启时共享的 Pipeline 状态 -// 分段重启时新旧 recorder 通过指针重定向共享同一份状态， +// 分段重启时新旧 recorder 通过 redirect 链共享同一份状态， // 确保旧任务的回调能正确递减新 recorder 的计数器 type pipelineSharedState struct { mu sync.Mutex @@ -277,6 +277,27 @@ type pipelineSharedState struct { details []notify.RecordingFileDetail // 收集的文件详情 sourceNames map[string]bool // 被 Pipeline 消费的原始文件名（用于合并时排除已删除的源文件） onAllTasksDone func() // 所有任务完成时的回调（分段重启时设置） + suppressSummary bool // 为 true 时，run() 退出不推送摘要（分段重启场景），用 mu 保护 + redirectedTo *pipelineSharedState // 分段重启时重定向到新 recorder 的共享状态 +} + +// resolveState 沿 redirect 链找到当前活跃的 pipelineSharedState。 +// 使用无锁遍历 + 锁内验证的方式，避免与 TransferPipelineState 产生死锁。 +// 返回时调用者持有 result.mu 锁。 +func resolveState(state *pipelineSharedState) *pipelineSharedState { + for { + // 无锁遍历找到链尾候选 + final := state + for final.redirectedTo != nil { + final = final.redirectedTo + } + // 锁住候选状态，验证没有被并发重定向 + final.mu.Lock() + if final.redirectedTo == nil { + return final // 调用者持有 final.mu 锁 + } + final.mu.Unlock() + } } type recorder struct { @@ -318,8 +339,7 @@ type recorder struct { // done 在 run() 退出时关闭，用于 CloseForRestart 等待 goroutine 完成 done chan struct{} - // suppressSummary 为 true 时，run() 退出不推送摘要（分段重启场景） - suppressSummary bool + // suppressSummary 已移入 pipelineSharedState，用 pipelineState.mu 保护 retryLogMu sync.Mutex lastRetryLogKey string @@ -1011,15 +1031,18 @@ func (r *recorder) sendAccumulatedSummary() { } // 标记摘要已发送（在实际发送前，防止并发双发） r.pipelineState.summarySent = true + suppressSummary := r.pipelineState.suppressSummary + pipelineDetails := r.pipelineState.details + pipelineSourceNames := r.pipelineState.sourceNames r.pipelineState.mu.Unlock() r.recordedFilesMu.Lock() defer r.recordedFilesMu.Unlock() - if r.suppressSummary { + if suppressSummary { r.getLogger().Infof("录制摘要推送已抑制（分段重启），累积 %d 个文件将传递给新 recorder", len(r.recordedFiles)) return } - if len(r.recordedFiles) == 0 && len(r.pipelineState.details) == 0 { + if len(r.recordedFiles) == 0 && len(pipelineDetails) == 0 { r.getLogger().Info("无录制文件，跳过摘要推送") return } @@ -1034,16 +1057,16 @@ func (r *recorder) sendAccumulatedSummary() { } // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 - if len(r.pipelineState.details) > 0 { + if len(pipelineDetails) > 0 { // 合并未被 Pipeline 覆盖的录制文件（如某分段因 stages==0 未入队） - merged := r.pipelineState.details + merged := pipelineDetails if len(r.recordedFiles) > 0 { inPipeline := map[string]bool{} - for _, d := range r.pipelineState.details { + for _, d := range pipelineDetails { inPipeline[d.Name] = true } // 排除已被 Pipeline 转换并删除的源文件（如 convert_mp4 delete_source 场景） - for name := range r.pipelineState.sourceNames { + for name := range pipelineSourceNames { inPipeline[name] = true } for _, f := range r.recordedFiles { @@ -1087,8 +1110,12 @@ func (r *recorder) sendAccumulatedSummary() { // onPipelineTaskComplete Pipeline 任务完成回调 // 由 Pipeline Manager 在 executeTask 结束时调用 func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { - r.pipelineState.mu.Lock() - r.pipelineState.pendingCount-- + // 沿 redirect 链找到当前活跃的共享状态 + // 支持连续重启 A→B→C 场景：A 的回调通过 resolveState 找到 C 的状态 + ps := resolveState(r.pipelineState) + // ps.mu 已被 resolveState 锁住 + + ps.pendingCount-- // 收集任务的最终文件详情 switch task.Status { @@ -1098,58 +1125,60 @@ func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { // 所有文件已上传并删除：从 LastStageFiles 提取真正上传的文件 uploadedDetails := extractUploadedDetails(task.LastStageFiles) if len(uploadedDetails) > 0 { - r.pipelineState.details = append(r.pipelineState.details, uploadedDetails...) + ps.details = append(ps.details, uploadedDetails...) } else { // 回退：无上传标记时用原始文件 for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { d.Uploaded = true - r.pipelineState.details = append(r.pipelineState.details, d) + ps.details = append(ps.details, d) } } } else { - r.pipelineState.details = append(r.pipelineState.details, pipeline.FileInfoToDetails(finalFiles)...) + ps.details = append(ps.details, pipeline.FileInfoToDetails(finalFiles)...) } case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled: // 失败/取消时优先用 CurrentFiles（最后成功阶段的输出），回退到 InitialFiles if len(task.CurrentFiles) > 0 { - r.pipelineState.details = append(r.pipelineState.details, pipeline.FileInfoToDetails(task.CurrentFiles)...) + ps.details = append(ps.details, pipeline.FileInfoToDetails(task.CurrentFiles)...) } else { - r.pipelineState.details = append(r.pipelineState.details, pipeline.FileInfoToDetails(task.InitialFiles)...) + ps.details = append(ps.details, pipeline.FileInfoToDetails(task.InitialFiles)...) } } // 收集被 Pipeline 转换的原始文件名（SourcePath），用于摘要合并时排除已删除的源文件 - collectSourceNames(r.pipelineState.sourceNames, task.CurrentFiles, task.LastStageFiles) + collectSourceNames(ps.sourceNames, task.CurrentFiles, task.LastStageFiles) - remaining := r.pipelineState.pendingCount - runExited := r.pipelineState.runExited - onAllDone := r.pipelineState.onAllTasksDone + remaining := ps.pendingCount + runExited := ps.runExited + onAllDone := ps.onAllTasksDone - r.pipelineState.mu.Unlock() + ps.mu.Unlock() r.getLogger().Infof("Pipeline 任务完成（状态: %s），剩余 %d 个任务", task.Status, remaining) // 所有任务完成且录制已结束，发送统一摘要 // 使用 goroutine 避免通知发送（Telegram/Email 等）阻塞 Pipeline 任务槽位 if remaining == 0 { - if r.suppressSummary { - // 分段重启场景：旧 recorder 已抑制，通过共享回调触发新 recorder 的摘要 - if onAllDone != nil { - onAllDone() - } + if onAllDone != nil { + // 分段重启场景：通过共享回调触发摘要（回调内部检查 runExited） + onAllDone() } else if runExited { - // run() 已退出，异步发送摘要（不阻塞 executeTask） + // 非分段重启场景：run() 已退出，异步发送摘要 go r.sendAccumulatedSummary() } - // run() 仍在运行且未抑制：等待 run() 退出时由 defer 中的 sendAccumulatedSummary 发送 + // run() 仍在运行且无回调：等待 run() 退出时由 defer 中的 sendAccumulatedSummary 发送 } } // extractUploadedDetails 从最后阶段输出文件中提取有上传标记的文件详情 // CloudUploadStage 会为成功上传的文件设置 Metadata["uploaded"]=true +// 排除封面文件（封面不在录制摘要中展示，与 FileInfoToDetails 保持一致） func extractUploadedDetails(files []pipeline.FileInfo) []notify.RecordingFileDetail { var details []notify.RecordingFileDetail for _, f := range files { + if f.Type == pipeline.FileTypeCover { + continue + } if f.Metadata != nil { if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { if f.Size > 0 { @@ -1310,9 +1339,9 @@ func (r *recorder) Close() { } func (r *recorder) CloseForRestart() []notify.RecordingFileDetail { - r.recordedFilesMu.Lock() - r.suppressSummary = true - r.recordedFilesMu.Unlock() + r.pipelineState.mu.Lock() + r.pipelineState.suppressSummary = true + r.pipelineState.mu.Unlock() r.Close() <-r.done // 等待 run() 完全退出，确保最后一个文件已累积 r.recordedFilesMu.Lock() @@ -1335,36 +1364,51 @@ func (r *recorder) SetInitialRecordedFiles(files []notify.RecordingFileDetail) { // TransferPipelineState 从旧 recorder 转移 Pipeline 状态 // 分段重启时由 RestartRecorder 调用，确保旧分段的 Pipeline 结果不丢失 +// 使用 redirect 链替代指针覆盖，支持连续重启 A→B→C 场景 func (r *recorder) TransferPipelineState(old *recorder) { r.pipelineState.mu.Lock() defer r.pipelineState.mu.Unlock() - old.pipelineState.mu.Lock() - defer old.pipelineState.mu.Unlock() + // 沿 old 的 redirect 链找到最终活跃状态（可能经过多次重启 A→B→C→...） + oldState := resolveState(old.pipelineState) + // 此时 oldState.mu 已被 resolveState 锁住 // 合并 Pipeline 状态 - r.pipelineState.enqueued = r.pipelineState.enqueued || old.pipelineState.enqueued - r.pipelineState.pendingCount += old.pipelineState.pendingCount + r.pipelineState.enqueued = r.pipelineState.enqueued || oldState.enqueued + r.pipelineState.pendingCount += oldState.pendingCount + // 注意：suppressSummary 不合并。该标志表示"我是被重启的 recorder，不发摘要"， + // 新 recorder 从未调用 CloseForRestart，应保持自己的 suppressSummary=false。 + // 回调通过 onAllTasksDone（含 runExited 检查）触发，不依赖 suppressSummary 传播。 // 合并文件详情（旧 recorder 的详情追加到新 recorder） - r.pipelineState.details = append(r.pipelineState.details, old.pipelineState.details...) + r.pipelineState.details = append(r.pipelineState.details, oldState.details...) // 合并已被 Pipeline 消费的原始文件名 - for name := range old.pipelineState.sourceNames { + for name := range oldState.sourceNames { r.pipelineState.sourceNames[name] = true } - // 关键：将旧 recorder 的 pipelineState 指针重定向到新 recorder 的共享状态 - // 这样旧任务完成时的回调会操作新 recorder 的计数器 - old.pipelineState = r.pipelineState - - // 设置回调：旧任务全部完成时，由旧回调触发新 recorder 的摘要发送 - if old.suppressSummary { - r.pipelineState.onAllTasksDone = func() { + // 设置回调：旧任务全部完成时，触发摘要发送 + // 回调与 suppressSummary 解耦：suppressSummary 是 recorder 级别的抑制标志， + // 回调是 pipeline 级别的完成通知。多跳 A→B→C 场景下，A 的回调通过 redirect + // 链解析到 C 的状态，如果只在 suppressSummary=true 时设置回调，B→C 转移时 + // B.suppressSummary=false 导致 C 没有回调，A 的摘要永远发不出。 + // runExited 检查确保只在新 recorder 已退出时发送，避免录制中提前发出摘要。 + r.pipelineState.onAllTasksDone = func() { + r.pipelineState.mu.Lock() + shouldSend := r.pipelineState.runExited + r.pipelineState.mu.Unlock() + if shouldSend { go r.sendAccumulatedSummary() } + // run() 未退出时不发送：新 recorder 的 defer 会在退出时检查并发送 } + // 关键：设置 redirect 链，使旧 recorder 及更早的 recorder 的回调 + // 能通过 resolveState 找到当前活跃的共享状态 + oldState.redirectedTo = r.pipelineState + oldState.mu.Unlock() + r.getLogger().Infof("继承 Pipeline 状态：pending=%d, details=%d 个文件", r.pipelineState.pendingCount, len(r.pipelineState.details)) } diff --git a/src/recorders/recorder_test.go b/src/recorders/recorder_test.go index f0899c9..7ea652a 100644 --- a/src/recorders/recorder_test.go +++ b/src/recorders/recorder_test.go @@ -11,9 +11,14 @@ import ( gomock "go.uber.org/mock/gomock" "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" livemock "github.com/bililive-go/bililive-go/src/live/mock" + "github.com/bililive-go/bililive-go/src/notify" + "github.com/bililive-go/bililive-go/src/pipeline" + "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" + "github.com/bililive-go/bililive-go/src/types" ) func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) { @@ -50,3 +55,331 @@ func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) { t.Fatalf("日志未保留原始模板错误: %s", logs) } } + +// newTestRecorder 创建用于测试的 recorder 实例（不启动 run goroutine） +func newTestRecorder(t *testing.T, liveId types.LiveID) *recorder { + t.Helper() + ctrl := gomock.NewController(t) + l := livemock.NewMockLive(ctrl) + l.EXPECT().GetLiveId().Return(liveId).AnyTimes() + l.EXPECT().GetLogger().Return(livelogger.New(0, logrus.Fields{"test": t.Name()})).AnyTimes() + + // 创建带 instance 的 context（NewRecorder 需要 instance.Cache 和 EventDispatcher） + inst := &instance.Instance{ + Cache: gcache.New(1).LRU().Build(), + } + ctx := context.WithValue(context.Background(), instance.Key, inst) + // NewDispatcher 会自动设置 inst.EventDispatcher + events.NewDispatcher(ctx) + + r, err := NewRecorder(ctx, l) + if err != nil { + t.Fatalf("创建 recorder 失败: %v", err) + } + return r.(*recorder) +} + +func TestResolveState_NoRedirect(t *testing.T) { + state := &pipelineSharedState{ + sourceNames: make(map[string]bool), + pendingCount: 5, + } + + resolved := resolveState(state) + if resolved != state { + t.Fatal("无 redirect 时 resolveState 应返回原始状态") + } + if resolved.pendingCount != 5 { + t.Fatalf("pendingCount 期望 5，实际 %d", resolved.pendingCount) + } + resolved.mu.Unlock() +} + +func TestResolveState_SingleRedirect(t *testing.T) { + stateA := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 3} + stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 0} + + // 模拟 A→B 重定向 + stateA.redirectedTo = stateB + + resolved := resolveState(stateA) + if resolved != stateB { + t.Fatal("单跳 redirect 应解析到 stateB") + } + resolved.mu.Unlock() +} + +func TestResolveState_MultiHopChain(t *testing.T) { + stateA := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 3} + stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 2} + stateC := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 1} + + // 模拟 A→B→C 链 + stateA.redirectedTo = stateB + stateB.redirectedTo = stateC + + resolved := resolveState(stateA) + if resolved != stateC { + t.Fatal("多跳 redirect 链应解析到 stateC") + } + resolved.mu.Unlock() +} + +func TestTransferPipelineState_ChainRedirect(t *testing.T) { + // 模拟 A→B→C 连续分段重启 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + recC := newTestRecorder(t, "test-live") + + // A 有 3 个待处理任务和一些文件详情 + recA.pipelineState.pendingCount = 3 + recA.pipelineState.enqueued = true + recA.pipelineState.details = []notify.RecordingFileDetail{ + {Name: "a.flv", Size: 100}, + } + recA.pipelineState.sourceNames["a_src.flv"] = true + // A 模拟已抑制（CloseForRestart 设置） + recA.pipelineState.suppressSummary = true + + // A → B 重启 + recB.TransferPipelineState(recA) + + // 验证 B 继承了 A 的状态 + if recB.pipelineState.pendingCount != 3 { + t.Fatalf("B pendingCount 期望 3，实际 %d", recB.pipelineState.pendingCount) + } + if len(recB.pipelineState.details) != 1 { + t.Fatalf("B details 期望 1，实际 %d", len(recB.pipelineState.details)) + } + + // B 有额外的 2 个任务 + recB.pipelineState.pendingCount += 2 // 模拟 B 自己又入队了 2 个任务 + + // B → C 重启 + recC.TransferPipelineState(recB) + + // 验证 C 继承了所有状态 + if recC.pipelineState.pendingCount != 5 { + t.Fatalf("C pendingCount 期望 5，实际 %d", recC.pipelineState.pendingCount) + } + + // 关键验证：A 的 pipelineState 通过 redirect 链应解析到 C 的状态 + resolvedA := resolveState(recA.pipelineState) + if resolvedA != recC.pipelineState { + t.Fatal("A 的状态应通过 redirect 链解析到 C 的状态") + } + resolvedA.mu.Unlock() + + // B 的 pipelineState 应直接指向 C + resolvedB := resolveState(recB.pipelineState) + if resolvedB != recC.pipelineState { + t.Fatal("B 的状态应解析到 C 的状态") + } + resolvedB.mu.Unlock() +} + +func TestOnPipelineTaskComplete_AfterChainRedirect(t *testing.T) { + // 模拟 A→B→C 后，A 的回调操作 C 的状态 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + recC := newTestRecorder(t, "test-live") + + // A 有 1 个待处理任务 + recA.pipelineState.pendingCount = 1 + recA.pipelineState.enqueued = true + recA.pipelineState.suppressSummary = true + recA.pipelineState.runExited = true + + // A → B → C + recB.TransferPipelineState(recA) + recC.TransferPipelineState(recB) + + // 模拟 C 的 run() 已退出（C 是当前活跃 recorder，runExited 是 per-recorder 的） + recC.pipelineState.mu.Lock() + recC.pipelineState.runExited = true + recC.pipelineState.mu.Unlock() + + // C 的 onAllTasksDone 回调应存在（TransferPipelineState 始终设置） + onAllDoneCalled := false + recC.pipelineState.onAllTasksDone = func() { + recC.pipelineState.mu.Lock() + shouldSend := recC.pipelineState.runExited + recC.pipelineState.mu.Unlock() + if shouldSend { + onAllDoneCalled = true + } + } + + // 模拟 A 的 Pipeline 任务完成 + task := &pipeline.PipelineTask{ + Status: pipeline.PipelineStatusCompleted, + CurrentFiles: []pipeline.FileInfo{}, + InitialFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo}, + }, + LastStageFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo, Metadata: map[string]any{"uploaded": true}}, + }, + } + + recA.onPipelineTaskComplete(task) + + // 验证：C 的 pendingCount 应为 0（A 的回调正确递减了 C 的计数） + if recC.pipelineState.pendingCount != 0 { + t.Fatalf("C pendingCount 期望 0，实际 %d（A 的回调未正确操作 C 的状态）", recC.pipelineState.pendingCount) + } + + // 验证：A 的回调收集的 details 应在 C 的状态中 + if len(recC.pipelineState.details) == 0 { + t.Fatal("C 的 details 不应为空（A 的回调应将文件详情写入 C 的状态）") + } + + // 验证：onAllTasksDone 应被触发（C 已退出） + if !onAllDoneCalled { + t.Fatal("onAllTasksDone 回调应被触发（C.runExited=true）") + } +} + +func TestExtractUploadedDetails_ExcludesCover(t *testing.T) { + files := []pipeline.FileInfo{ + {Path: "/data/video.flv", Size: 1024, Type: pipeline.FileTypeVideo, Metadata: map[string]any{"uploaded": true}}, + {Path: "/data/cover.jpg", Size: 512, Type: pipeline.FileTypeCover, Metadata: map[string]any{"uploaded": true}}, + {Path: "/data/other.txt", Size: 256, Type: pipeline.FileTypeOther, Metadata: map[string]any{"uploaded": true}}, + } + + details := extractUploadedDetails(files) + + if len(details) != 2 { + t.Fatalf("期望 2 个文件（排除封面），实际 %d", len(details)) + } + + names := map[string]bool{} + for _, d := range details { + names[d.Name] = true + } + if names["cover.jpg"] { + t.Fatal("封面文件不应出现在上传详情中") + } + if !names["video.flv"] || !names["other.txt"] { + t.Fatal("视频和其他类型文件应出现在上传详情中") + } +} + +func TestSuppressSummary_InSharedState(t *testing.T) { + // 验证 suppressSummary 在 pipelineSharedState 中，通过 mu 保护 + rec := newTestRecorder(t, "test-live") + + // 初始值应为 false + rec.pipelineState.mu.Lock() + if rec.pipelineState.suppressSummary { + t.Fatal("suppressSummary 初始值应为 false") + } + rec.pipelineState.mu.Unlock() + + // CloseForRestart 设置 suppressSummary + rec.pipelineState.mu.Lock() + rec.pipelineState.suppressSummary = true + rec.pipelineState.mu.Unlock() + + // 验证设置成功 + rec.pipelineState.mu.Lock() + if !rec.pipelineState.suppressSummary { + t.Fatal("suppressSummary 应为 true") + } + rec.pipelineState.mu.Unlock() +} + +func TestTransferPipelineState_SetRedirect(t *testing.T) { + // 验证 TransferPipelineState 设置了 redirect 链 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + + recA.pipelineState.pendingCount = 1 + recA.pipelineState.suppressSummary = true + + recB.TransferPipelineState(recA) + + // A 的状态应被重定向到 B + if recA.pipelineState.redirectedTo != recB.pipelineState { + t.Fatal("A 的 redirect 应指向 B 的状态") + } + + // resolveState 应从 A 找到 B + resolved := resolveState(recA.pipelineState) + if resolved != recB.pipelineState { + t.Fatal("resolveState(A) 应返回 B 的状态") + } + resolved.mu.Unlock() +} + +func TestOnAllTasksDone_NotSentWhileRecording(t *testing.T) { + // 验证 Bot 指出的场景：旧任务完成时新 recorder 仍在录制，摘要不应提前发送 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + + recA.pipelineState.pendingCount = 1 + recA.pipelineState.enqueued = true + recA.pipelineState.suppressSummary = true + recA.pipelineState.runExited = true + + // A → B + recB.TransferPipelineState(recA) + + // B 仍在录制（runExited=false） + // 记录 B 的 onAllTasksDone 是否被调用 + callbackTriggered := false + originalCallback := recB.pipelineState.onAllTasksDone + recB.pipelineState.onAllTasksDone = func() { + callbackTriggered = true + originalCallback() + } + + // 模拟 A 的任务完成 + task := &pipeline.PipelineTask{ + Status: pipeline.PipelineStatusCompleted, + CurrentFiles: []pipeline.FileInfo{}, + InitialFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo}, + }, + LastStageFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo}, + }, + } + recA.onPipelineTaskComplete(task) + + // 回调应被触发（remaining=0, suppressSummary=true） + if !callbackTriggered { + t.Fatal("onAllTasksDone 应被触发") + } + + // 但摘要不应被发送（B 仍在录制，runExited=false） + // 验证方式：summarySent 应仍为 false + recB.pipelineState.mu.Lock() + summarySent := recB.pipelineState.summarySent + recB.pipelineState.mu.Unlock() + if summarySent { + t.Fatal("摘要不应在新 recorder 仍在录制时发送（summarySent 应为 false）") + } +} + +func TestFinalRecorder_SummaryNotSuppressed(t *testing.T) { + // 验证最终 recorder 的摘要不被 suppressSummary 抑制 + // （suppressSummary 不应从旧 recorder 传播到新 recorder） + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + + recA.pipelineState.pendingCount = 1 + recA.pipelineState.enqueued = true + recA.pipelineState.suppressSummary = true + + // A → B + recB.TransferPipelineState(recA) + + // B 的 suppressSummary 应为 false（不从 A 继承） + recB.pipelineState.mu.Lock() + bSuppress := recB.pipelineState.suppressSummary + recB.pipelineState.mu.Unlock() + if bSuppress { + t.Fatal("最终 recorder 的 suppressSummary 应为 false（不应从旧 recorder 继承）") + } +} From 6fde121e5547e57e5763e3f4da4d415618d06ee8 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:34:59 +0800 Subject: [PATCH 29/52] =?UTF-8?q?fix(pipeline,recorders):=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=88=86=E6=AE=B5=E9=87=8D=E5=90=AF=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E7=9A=84=E4=BA=94=E4=B8=AA=E5=B9=B6=E5=8F=91=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. redirectedTo 数据竞争：改用 atomic.Pointer，遍历 Load()、写入 Store() 2. merged 底层数组写冲突：快照时深拷贝，断开与 ps.details 的共享 3. 摘要发送先于状态转移：transferWg 屏障确保 Pipeline 状态转入后再发送 4. 回调注册先于任务可见：扩展 m.mu.Lock 覆盖 CreateTask + 回调注册， 阻塞 pollLoop 的 scheduleNextTasks 5. 异步通知 goroutine 未跟踪：notifyWg 跟踪 go sendAccumulatedSummary， 关闭流程中 PipelineManager.Close 后等待通知完成 Co-Authored-By: Kscc --- src/cmd/bililive/bililive.go | 5 ++++ src/pipeline/manager.go | 11 +++---- src/recorders/manager.go | 28 +++++++++++++++--- src/recorders/recorder.go | 54 ++++++++++++++++++++++++++++------ src/recorders/recorder_test.go | 8 ++--- 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/cmd/bililive/bililive.go b/src/cmd/bililive/bililive.go index 9def416..e511e57 100644 --- a/src/cmd/bililive/bililive.go +++ b/src/cmd/bililive/bililive.go @@ -826,6 +826,11 @@ func main() { if inst.PipelineManager != nil { inst.PipelineManager.Close(ctx) } + // 等待异步通知 goroutine（如 Telegram/Email 摘要发送）完成 + // 必须在 PipelineManager.Close 之后——pipeline 回调可能触发最后的通知 + if rm, ok := inst.RecorderManager.(recorders.Manager); ok { + rm.WaitForNotifications() + } // 关闭直播间状态管理器 if liveStateManager != nil { liveStateManager.Close() diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 4f99eaa..4be7926 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -319,18 +319,19 @@ func FileInfoToDetails(files []FileInfo) []notify.RecordingFileDetail { } // EnqueueTask 添加任务到队列 -// onTaskComplete: 任务完成回调（可选），在异步调度前注册，避免竞态窗口 +// onTaskComplete: 任务完成回调（可选），在任务对调度器可见前注册 func (m *Manager) EnqueueTask(task *PipelineTask, onTaskComplete func(*PipelineTask)) error { + // 持锁覆盖 CreateTask + 回调注册，阻塞 pollLoop 的 scheduleNextTasks（需 RLock） + // 确保任务对调度器可见时回调已就绪 + m.mu.Lock() if err := m.store.CreateTask(m.ctx, task); err != nil { + m.mu.Unlock() return fmt.Errorf("failed to create pipeline task: %w", err) } - - // 在异步调度前注册回调，确保 executeTask 读取时回调已就绪 if onTaskComplete != nil { - m.mu.Lock() m.taskCallbacks[task.ID] = onTaskComplete - m.mu.Unlock() } + m.mu.Unlock() logrus.WithFields(logrus.Fields{ "task_id": task.ID, diff --git a/src/recorders/manager.go b/src/recorders/manager.go index 3e43a5b..361a44b 100644 --- a/src/recorders/manager.go +++ b/src/recorders/manager.go @@ -73,6 +73,8 @@ type Manager interface { GetRecorderStatus(ctx context.Context, liveId types.LiveID) (map[string]interface{}, error) // GetActiveRecordingsCount 获取当前活跃的录制数量 GetActiveRecordingsCount() int + // WaitForNotifications 等待所有异步通知 goroutine 完成（关闭流程中使用） + WaitForNotifications() } // for test @@ -86,6 +88,7 @@ type manager struct { statusTicker *time.Ticker statusStopCh chan struct{} statusWg sync.WaitGroup // 用于等待广播 goroutine 退出 + notifyWg sync.WaitGroup // 跟踪异步通知 goroutine（sendAccumulatedSummary），关闭时等待 // restartingCount 追踪正在执行 CloseForRestart 的旧 recorder 数量。 // RestartRecorder 在释放锁后才执行 oldRecorder.CloseForRestart()， // 此期间 map 中只有新 recorder，但旧 recorder 仍在收尾运行。 @@ -169,6 +172,12 @@ func (m *manager) Close(ctx context.Context) { inst.WaitGroup.Done() } +// WaitForNotifications 等待所有异步通知 goroutine 完成 +// 应在 PipelineManager.Close 之后调用，确保所有 pipeline 回调触发的通知已完成 +func (m *manager) WaitForNotifications() { + m.notifyWg.Wait() +} + func (m *manager) AddRecorder(ctx context.Context, live live.Live) error { m.lock.Lock() defer m.lock.Unlock() @@ -180,11 +189,15 @@ func (m *manager) addRecorderLocked(ctx context.Context, live live.Live) error { if _, ok := m.savers[live.GetLiveId()]; ok { return ErrRecorderExist } - recorder, err := newRecorder(ctx, live) + rec, err := newRecorder(ctx, live) if err != nil { return err } - m.savers[live.GetLiveId()] = recorder + // 设置通知 WaitGroup，使异步摘要发送 goroutine 可被关闭流程等待 + if recTyped, ok := rec.(*recorder); ok { + recTyped.notifyWg = &m.notifyWg + } + m.savers[live.GetLiveId()] = rec cfg := configs.GetCurrentConfig() if cfg != nil { @@ -192,12 +205,12 @@ func (m *manager) addRecorderLocked(ctx context.Context, live live.Live) error { bilisentry.GoWithContext(ctx, func(ctx context.Context) { m.cronRestart(ctx, live) }) } } - if err := recorder.Start(ctx); err != nil { + if err := rec.Start(ctx); err != nil { // Start 失败时从 map 删除并异步 Close 新 recorder，防止泄漏/僵尸实例 // 使用异步 Close 避免在持锁时执行耗时操作（如等待 ffmpeg 进程退出）， // 防止长时间阻塞其他 manager 操作 delete(m.savers, live.GetLiveId()) - bilisentry.Go(recorder.Close) + bilisentry.Go(rec.Close) return err } return nil @@ -241,6 +254,11 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error { return err } newRec := m.savers[live.GetLiveId()] + // 设置分段重启摘要屏障：阻止新 recorder 在 Pipeline 状态转移前发送摘要 + if newRecTyped, ok := newRec.(*recorder); ok { + newRecTyped.transferWg = &sync.WaitGroup{} + newRecTyped.transferWg.Add(1) + } // restartingCount 必须在释放锁之前递增，否则 Unlock 到 Add(1) 之间 // LiveEnd 可能移除新 recorder 并看到 restartingCount==0， // 导致 GetActiveRecordingsCount() 误判为"无活跃录制"触发优雅更新 @@ -279,6 +297,8 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error { if oldRecTyped, ok := oldRecorder.(*recorder); ok { newRecTyped.TransferPipelineState(oldRecTyped) } + // 释放摘要屏障：转移已完成（或无需转移），允许新 recorder 发送摘要 + newRecTyped.transferWg.Done() } return nil diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index babafe4..a056be7 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -278,7 +278,7 @@ type pipelineSharedState struct { sourceNames map[string]bool // 被 Pipeline 消费的原始文件名（用于合并时排除已删除的源文件） onAllTasksDone func() // 所有任务完成时的回调（分段重启时设置） suppressSummary bool // 为 true 时，run() 退出不推送摘要（分段重启场景），用 mu 保护 - redirectedTo *pipelineSharedState // 分段重启时重定向到新 recorder 的共享状态 + redirectedTo atomic.Pointer[pipelineSharedState] // 分段重启时重定向到新 recorder 的共享状态 } // resolveState 沿 redirect 链找到当前活跃的 pipelineSharedState。 @@ -286,14 +286,14 @@ type pipelineSharedState struct { // 返回时调用者持有 result.mu 锁。 func resolveState(state *pipelineSharedState) *pipelineSharedState { for { - // 无锁遍历找到链尾候选 + // 无锁遍历找到链尾候选（通过 atomic.Load 避免与写入端的数据竞争） final := state - for final.redirectedTo != nil { - final = final.redirectedTo + for next := final.redirectedTo.Load(); next != nil; next = final.redirectedTo.Load() { + final = next } // 锁住候选状态，验证没有被并发重定向 final.mu.Lock() - if final.redirectedTo == nil { + if final.redirectedTo.Load() == nil { return final // 调用者持有 final.mu 锁 } final.mu.Unlock() @@ -320,6 +320,17 @@ type recorder struct { // 使用指针，分段重启时新旧 recorder 共享同一份状态 pipelineState *pipelineSharedState + // transferWg 用于分段重启时的摘要屏障。 + // RestartRecorder 在创建新 recorder 后 Add(1)， + // TransferPipelineState 完成后 Done()。 + // sendAccumulatedSummary 在发送前 Wait()，确保旧分段状态已转入。 + // 非分段重启场景为 nil，Wait 是 no-op。 + transferWg *sync.WaitGroup + + // notifyWg 跟踪异步通知 goroutine（go sendAccumulatedSummary）。 + // 由 RecorderManager 持有，关闭流程中等待所有通知发送完成。 + notifyWg *sync.WaitGroup + // 当前录制的流信息（来自平台 API） currentStreamInfo *live.AvailableStreamInfo @@ -1016,6 +1027,11 @@ func (r *recorder) accumulateRecordedFiles(files ...string) { // sendAccumulatedSummary 录制结束后统一推送录制文件摘要通知 // 在 run() 退出时通过 defer 调用，确保所有分段文件汇总为一条通知 func (r *recorder) sendAccumulatedSummary() { + // 分段重启屏障：等待 TransferPipelineState 完成， + // 确保旧分段的 pipelineDetails/pendingCount 已转入再发送摘要 + if r.transferWg != nil { + r.transferWg.Wait() + } r.pipelineState.mu.Lock() pendingCount := r.pipelineState.pendingCount // 幂等保护：防止 defer 和回调并发双发 @@ -1059,7 +1075,7 @@ func (r *recorder) sendAccumulatedSummary() { // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 if len(pipelineDetails) > 0 { // 合并未被 Pipeline 覆盖的录制文件（如某分段因 stages==0 未入队） - merged := pipelineDetails + merged := append([]notify.RecordingFileDetail(nil), pipelineDetails...) if len(r.recordedFiles) > 0 { inPipeline := map[string]bool{} for _, d := range pipelineDetails { @@ -1164,7 +1180,17 @@ func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { onAllDone() } else if runExited { // 非分段重启场景：run() 已退出，异步发送摘要 - go r.sendAccumulatedSummary() + if r.notifyWg != nil { + r.notifyWg.Add(1) + } + go func() { + defer func() { + if r.notifyWg != nil { + r.notifyWg.Done() + } + }() + r.sendAccumulatedSummary() + }() } // run() 仍在运行且无回调：等待 run() 退出时由 defer 中的 sendAccumulatedSummary 发送 } @@ -1399,14 +1425,24 @@ func (r *recorder) TransferPipelineState(old *recorder) { shouldSend := r.pipelineState.runExited r.pipelineState.mu.Unlock() if shouldSend { - go r.sendAccumulatedSummary() + if r.notifyWg != nil { + r.notifyWg.Add(1) + } + go func() { + defer func() { + if r.notifyWg != nil { + r.notifyWg.Done() + } + }() + r.sendAccumulatedSummary() + }() } // run() 未退出时不发送：新 recorder 的 defer 会在退出时检查并发送 } // 关键：设置 redirect 链，使旧 recorder 及更早的 recorder 的回调 // 能通过 resolveState 找到当前活跃的共享状态 - oldState.redirectedTo = r.pipelineState + oldState.redirectedTo.Store(r.pipelineState) oldState.mu.Unlock() r.getLogger().Infof("继承 Pipeline 状态：pending=%d, details=%d 个文件", diff --git a/src/recorders/recorder_test.go b/src/recorders/recorder_test.go index 7ea652a..aa2f88d 100644 --- a/src/recorders/recorder_test.go +++ b/src/recorders/recorder_test.go @@ -100,7 +100,7 @@ func TestResolveState_SingleRedirect(t *testing.T) { stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 0} // 模拟 A→B 重定向 - stateA.redirectedTo = stateB + stateA.redirectedTo.Store(stateB) resolved := resolveState(stateA) if resolved != stateB { @@ -115,8 +115,8 @@ func TestResolveState_MultiHopChain(t *testing.T) { stateC := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 1} // 模拟 A→B→C 链 - stateA.redirectedTo = stateB - stateB.redirectedTo = stateC + stateA.redirectedTo.Store(stateB) + stateB.redirectedTo.Store(stateC) resolved := resolveState(stateA) if resolved != stateC { @@ -300,7 +300,7 @@ func TestTransferPipelineState_SetRedirect(t *testing.T) { recB.TransferPipelineState(recA) // A 的状态应被重定向到 B - if recA.pipelineState.redirectedTo != recB.pipelineState { + if recA.pipelineState.redirectedTo.Load() != recB.pipelineState { t.Fatal("A 的 redirect 应指向 B 的状态") } From ad5fa50a59155e00c1145f3903ac0ff6400dc41c Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:06:08 +0800 Subject: [PATCH 30/52] =?UTF-8?q?fix(pipeline):=20=E4=BA=91=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E6=A8=A1=E5=BC=8F=E4=B8=8B=20.ass=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=9C=AA=E6=A0=87=E8=AE=B0=20uploaded=20=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E6=91=98=E8=A6=81=E8=AF=AF=E6=98=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cloud upload 的 deleteAllAfter 和 deleteAfter 模式都只处理视频文件， 关联的 .ass 字幕文件不在 pipeline 中，不会被标记 Metadata[uploaded]。 摘要将其作为本地文件显示，但实际已被删除。 修复： - deleteAllAfter：加入 output 时即设置 Metadata[uploaded] = true - deleteAfter：为已上传视频文件查找关联 .ass，加入 output 并标记 uploaded Co-Authored-By: Kscc --- src/pipeline/stages/extract_cover.go | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index ee4f90d..085f194 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -314,6 +314,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli output = append(output, pipeline.FileInfo{ Path: assPath, Type: pipeline.FileTypeOther, + Metadata: map[string]any{ + "uploaded": true, // 全部上传成功，字幕文件将一并删除，标记为已上传 + }, }) assAdded[assPath] = true ctx.Logger.Infof("delete_all 模式：关联字幕文件 %s", assPath) @@ -353,6 +356,34 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli count++ } } + // 已上传的视频文件会被删除，其关联的 .ass 字幕文件也应标记为已上传 + // 避免摘要将已删除的字幕文件误显为本地保留文件 + for _, f := range output { + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { + continue + } + assPath := strings.TrimSuffix(f.Path, ext) + ".ass" + if _, err := os.Stat(assPath); err == nil { + alreadyInOutput := false + for _, of := range output { + if of.Path == assPath { + alreadyInOutput = true + break + } + } + if !alreadyInOutput { + output = append(output, pipeline.FileInfo{ + Path: assPath, + Type: pipeline.FileTypeOther, + Metadata: map[string]any{ + "uploaded": true, + }, + }) + ctx.Logger.Infof("delete_uploaded 模式：关联字幕文件 %s", assPath) + } + } + } s.logs += fmt.Sprintf("全部上传成功，已标记 %d 个已上传文件待删除\n", count) ctx.Logger.Infof("全部上传成功，已标记 %d 个已上传文件待删除", count) } From f5716ad3dec9b1fea9229e7ef2c0422a1bdc72e0 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:42:27 +0800 Subject: [PATCH 31/52] =?UTF-8?q?feat(pipeline):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=20.ass=20=E5=BC=B9=E5=B9=95=E5=AD=97?= =?UTF-8?q?=E5=B9=95=E6=96=87=E4=BB=B6=E5=88=B0=E4=BA=91=E5=AD=98=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 upload_subtitles 配置项，启用后 cloud upload stage 会在上传循环前 从磁盘发现与视频文件同名的 .ass 字幕文件，加入 input 参与上传。 改动： - configs: CloudUpload 新增 UploadSubtitles 字段 - pipeline/config: 新增 OptionUploadSubtitles，ConvertLegacyConfig 传递 - extract_cover: CloudUploadStage 新增 uploadSubtitles 字段， Execute 中扫描 input 视频文件发现关联 .ass 并加入上传队列， 文件类型过滤放行 .ass 限制：仅 after_process 模式生效（immediate 模式下 .ass 尚未生成）。 需要同时开启弹幕录制（danmaku_enable）才有 .ass 文件可上传。 Co-Authored-By: Kscc --- src/configs/config.go | 1 + src/pipeline/config.go | 3 + src/pipeline/stages/extract_cover.go | 93 ++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 20 deletions(-) diff --git a/src/configs/config.go b/src/configs/config.go index 4a46402..d76302a 100644 --- a/src/configs/config.go +++ b/src/configs/config.go @@ -354,6 +354,7 @@ type CloudUpload struct { UploadPathTmpl string `yaml:"upload_path_tmpl" json:"upload_path_tmpl"` // 上传路径模板 DeleteAfterUpload bool `yaml:"delete_after_upload" json:"delete_after_upload"` // 上传成功后仅删除已上传的文件 DeleteAllAfterUpload bool `yaml:"delete_all_after_upload" json:"delete_all_after_upload"` // 上传成功后删除全部文件（含中间产物） + UploadSubtitles bool `yaml:"upload_subtitles" json:"upload_subtitles"` // 是否上传关联的 .ass 弹幕字幕文件 AdditionalStorages []string `yaml:"additional_storages,omitempty" json:"additional_storages,omitempty"` // 额外存储（支持多目标上传） } diff --git a/src/pipeline/config.go b/src/pipeline/config.go index 6c33549..77bf36a 100644 --- a/src/pipeline/config.go +++ b/src/pipeline/config.go @@ -32,6 +32,8 @@ const ( OptionCommand = "command" // OptionFileTypes 处理的文件类型过滤 OptionFileTypes = "file_types" + // OptionUploadSubtitles 是否上传关联的 .ass 弹幕字幕文件 + OptionUploadSubtitles = "upload_subtitles" // OptionCodec 视频编码器 OptionCodec = "codec" // OptionCrf CRF 质量值 @@ -85,6 +87,7 @@ func ConvertLegacyConfig(legacy *configs.OnRecordFinished) *PipelineConfig { OptionDeleteAllAfter: legacy.CloudUpload.DeleteAllAfterUpload, OptionUploadTiming: string(legacy.UploadTiming), OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)}, + OptionUploadSubtitles: legacy.CloudUpload.UploadSubtitles, }, } hasCloudUpload := legacy.CloudUpload.Enable && legacy.CloudUpload.StorageName != "" diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index 085f194..68fbb59 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -97,27 +97,29 @@ func (s *ExtractCoverStage) GetLogs() string { // CloudUploadStage 云上传阶段 type CloudUploadStage struct { - config pipeline.StageConfig - storageName string - pathTemplate string - deleteAfter bool // 仅删除已上传的文件 - deleteAllAfter bool // 删除全部文件（含中间产物） - uploadTiming string // immediate 或 after_process - fileTypes []string // 过滤的文件类型，空表示所有 - commands []string - logs string + config pipeline.StageConfig + storageName string + pathTemplate string + deleteAfter bool // 仅删除已上传的文件 + deleteAllAfter bool // 删除全部文件（含中间产物） + uploadTiming string // immediate 或 after_process + fileTypes []string // 过滤的文件类型，空表示所有 + uploadSubtitles bool // 是否上传关联的 .ass 弹幕字幕文件 + commands []string + logs string } // NewCloudUploadStage 创建云上传阶段工厂 func NewCloudUploadStage(config pipeline.StageConfig) (pipeline.Stage, error) { return &CloudUploadStage{ - config: config, - storageName: config.GetStringOption(pipeline.OptionStorage, ""), - pathTemplate: config.GetStringOption(pipeline.OptionPathTemplate, ""), - deleteAfter: config.GetBoolOption(pipeline.OptionDeleteAfter, false), - deleteAllAfter: config.GetBoolOption(pipeline.OptionDeleteAllAfter, false), - uploadTiming: config.GetStringOption(pipeline.OptionUploadTiming, ""), - fileTypes: config.GetStringSliceOption(pipeline.OptionFileTypes), + config: config, + storageName: config.GetStringOption(pipeline.OptionStorage, ""), + pathTemplate: config.GetStringOption(pipeline.OptionPathTemplate, ""), + deleteAfter: config.GetBoolOption(pipeline.OptionDeleteAfter, false), + deleteAllAfter: config.GetBoolOption(pipeline.OptionDeleteAllAfter, false), + uploadTiming: config.GetStringOption(pipeline.OptionUploadTiming, ""), + fileTypes: config.GetStringSliceOption(pipeline.OptionFileTypes), + uploadSubtitles: config.GetBoolOption(pipeline.OptionUploadSubtitles, false), }, nil } @@ -179,6 +181,52 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 与最终视频（如录播姬生成的多分段 _PART 文件），无需额外限制 uploadedIndices := map[int]bool{} // 记录实际上传的文件在 output 中的索引 + // 上传弹幕字幕：扫描 input 中的视频文件，从磁盘发现关联的 .ass 文件并加入 input + if s.uploadSubtitles { + assAdded := map[string]bool{} + var discovered []pipeline.FileInfo + for _, f := range input { + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { + continue + } + assPath := strings.TrimSuffix(f.Path, ext) + ".ass" + if assAdded[assPath] { + continue + } + if info, err := os.Stat(assPath); err == nil && info.Size() > 0 { + // 检查 input 和 discovered 中是否已存在 + alreadyExists := false + for _, of := range input { + if of.Path == assPath { + alreadyExists = true + break + } + } + if !alreadyExists { + for _, of := range discovered { + if of.Path == assPath { + alreadyExists = true + break + } + } + } + if !alreadyExists { + discovered = append(discovered, pipeline.FileInfo{ + Path: assPath, + Size: info.Size(), + Type: pipeline.FileTypeOther, + }) + assAdded[assPath] = true + ctx.Logger.Infof("发现关联弹幕字幕文件: %s", assPath) + } + } + } + if len(discovered) > 0 { + input = append(input, discovered...) + } + } + for _, file := range input { // 跳过标记为待删除的中间文件（由其他阶段产出，不应上传） if file.Deletable { @@ -186,10 +234,12 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli continue } - // 文件类型过滤 + // 文件类型过滤（uploadSubtitles 时放行 .ass 字幕文件） if len(s.fileTypes) > 0 && !s.matchFileType(file.Type) { - output = append(output, file) - continue + if !s.uploadSubtitles || strings.ToLower(filepath.Ext(file.Path)) != ".ass" { + output = append(output, file) + continue + } } // 检查文件是否存在 @@ -358,7 +408,10 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } // 已上传的视频文件会被删除，其关联的 .ass 字幕文件也应标记为已上传 // 避免摘要将已删除的字幕文件误显为本地保留文件 - for _, f := range output { + for i, f := range output { + if !uploadedIndices[i] { + continue // 仅处理已上传成功的视频文件 + } ext := strings.ToLower(filepath.Ext(f.Path)) if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { continue From 79426525e9266a5b0a92052fd821f187d3722042 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:48:06 +0800 Subject: [PATCH 32/52] =?UTF-8?q?feat(webapp):=20=E4=BA=91=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E6=96=B0=E5=A2=9E=E3=80=8C=E4=B8=8A=E4=BC=A0=E5=BC=B9?= =?UTF-8?q?=E5=B9=95=E5=AD=97=E5=B9=95=E3=80=8D=E5=BC=80=E5=85=B3=E5=8F=8A?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=84=E7=90=86=E9=A2=84=E8=A7=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - EffectiveConfig 接口新增 upload_subtitles 字段 - CloudUploadSettings 新增开关，位于删除开关之后、OpenList 认证之前 - analyzeConfig 预览逻辑：after_process 模式下 .ass 标记为 uploaded， immediate 模式不标记（.ass 尚未生成），兜底逻辑保持无视频时删除 .ass Co-Authored-By: Kscc --- .../component/config-info/CloudUploadSettings.tsx | 15 ++++++++++++++- src/webapp/src/component/config-info/index.tsx | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index a24a0f0..582791c 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -123,6 +123,7 @@ const analyzeConfig = (config: any) => { const upload = cu.enable ?? false; const delAfter = cu.delete_after_upload ?? false; const delAllAfter = cu.delete_all_after_upload ?? false; + const uploadAss = cu.upload_subtitles ?? false; // 文件状态：uploaded(上传), deleted(删除), kept(保留) const files: Record = {}; @@ -131,9 +132,12 @@ const analyzeConfig = (config: any) => { files['video.flv'] = { ext: '.flv', desc: '原始录制视频', status: 'kept' }; files['video.ass'] = { ext: '.ass', desc: '弹幕字幕', status: 'kept' }; - // immediate 模式：先上传原始文件 + // immediate 模式：先上传原始文件（.ass 在录制结束时已存在，可一并上传） if (isImmediate && upload) { files['video.flv'].status = 'uploaded'; + if (uploadAss) { + files['video.ass'].status = 'uploaded'; + } } // 转码 @@ -198,6 +202,10 @@ const analyzeConfig = (config: any) => { if (cover) { files['cover.jpg'].status = 'uploaded'; } + // 上传弹幕字幕（需 after_process 模式，immediate 模式下 .ass 尚未生成） + if (uploadAss) { + files['video.ass'].status = 'uploaded'; + } // 删除逻辑 if (delAllAfter) { @@ -444,6 +452,11 @@ const CloudUploadSettings: React.FC = ({ config, form + + + + + Date: Mon, 17 Aug 2026 15:09:27 +0800 Subject: [PATCH 33/52] =?UTF-8?q?fix(servers):=20=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=BC=B9=E5=B9=95=E5=AD=97=E5=B9=95=E5=BC=80=E5=85=B3=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E4=BF=9D=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handler.go 解析 cloud_upload 配置时遗漏 upload_subtitles 字段， 前端发送的开关值被后端忽略，导致保存无效。 Co-Authored-By: Kscc --- src/servers/handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/servers/handler.go b/src/servers/handler.go index 2cfd0b9..00371f7 100644 --- a/src/servers/handler.go +++ b/src/servers/handler.go @@ -1727,6 +1727,9 @@ func applyConfigUpdates(c *configs.Config, updates map[string]interface{}) error if deleteAllAfter, ok := cloudUpload["delete_all_after_upload"].(bool); ok { c.OnRecordFinished.CloudUpload.DeleteAllAfterUpload = deleteAllAfter } + if uploadSubtitles, ok := cloudUpload["upload_subtitles"].(bool); ok { + c.OnRecordFinished.CloudUpload.UploadSubtitles = uploadSubtitles + } } // 处理上传时机 if uploadTiming, ok := orf["upload_timing"].(string); ok { From cdd0544d793726e5258384c1bdc955f16d6d8074 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:18:08 +0800 Subject: [PATCH 34/52] =?UTF-8?q?fix(pipeline,servers):=20=E6=89=8B?= =?UTF-8?q?=E5=8A=A8=E4=B8=8A=E4=BC=A0=E5=92=8C=E9=85=8D=E7=BD=AE=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E9=81=97=E6=BC=8F=20upload=5Fsubtitles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. EnqueueUploadTask 构建 pipeline config 时未传递 OptionUploadSubtitles， 手动单文件/批量上传不会发现关联 .ass 文件 2. handler.go 解析 cloud_upload 配置时遗漏 upload_subtitles 字段， 前端开关保存无效 Co-Authored-By: Kscc --- src/pipeline/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index 4be7926..50d4395 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -672,6 +672,7 @@ func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped [] OptionDeleteAllAfter: cu.DeleteAllAfterUpload, OptionUploadTiming: "after_process", // 手动上传无后续处理阶段，始终允许删除标记（不继承全局 upload_timing） OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)}, + OptionUploadSubtitles: cu.UploadSubtitles, }, }, }, From 9eb367623a3ebbf8cc49537e97f9baa4f41e35f2 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:20:33 +0800 Subject: [PATCH 35/52] =?UTF-8?q?fix(notify):=20=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=90=8E=E4=B8=8D=E5=88=A0=E9=99=A4=E6=97=B6=E6=91=98=E8=A6=81?= =?UTF-8?q?=E4=B9=9F=E5=BA=94=E6=98=BE=E7=A4=BA=E5=B7=B2=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原逻辑把「已上传」等同于「已删除」，只有所有文件都上传且删除时才走 「已上传到云端」分支。上传但不删除时，通知错误显示「本地文件已清理」。 修复：分离 uploaded 和 kept 两个维度，支持三种通知格式： - 全部上传：显示「已上传到云端」 - 部分上传+部分保留：显示两段（上传列表+本地列表+磁盘空间） - 全部本地保留：显示文件列表+磁盘空间 Co-Authored-By: Kscc --- src/notify/send.go | 79 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/src/notify/send.go b/src/notify/send.go index 0539667..48bfbc0 100644 --- a/src/notify/send.go +++ b/src/notify/send.go @@ -274,6 +274,54 @@ func buildUploadedSummaryBody(platform string, files []RecordingFileDetail) stri return sb.String() } +// buildMixedSummaryBody 构造"部分上传、部分本地保留"场景的消息体 +func buildMixedSummaryBody(platform string, uploaded, kept []RecordingFileDetail, outputPath string) string { + const maxDisplayFiles = 30 + + var sb strings.Builder + fmt.Fprintf(&sb, "平台：%s\n", platform) + + // 已上传部分 + fmt.Fprintf(&sb, "已上传到云端：%d 个\n", len(uploaded)) + var uploadedSize int64 + for i, f := range uploaded { + uploadedSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(uploaded) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(uploaded)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "上传总大小：%s\n", formatFileSize(uploadedSize)) + + // 本地保留部分 + fmt.Fprintf(&sb, "本地保留：%d 个\n", len(kept)) + var keptSize int64 + for i, f := range kept { + keptSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(kept) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(kept)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "本地总大小：%s", formatFileSize(keptSize)) + if outputPath != "" { + if free, err := getDiskFreeSpace(outputPath); err == nil { + fmt.Fprintf(&sb, "\n剩余磁盘空间：%s", formatFileSize(int64(free))) + } + } + return sb.String() +} + +// buildRecordingSummaryMessageBody 构造纯本地保留的消息体（不含 title） +func buildRecordingSummaryMessageBody(platform string, files []RecordingFileDetail, outputPath string) string { + _, body := buildRecordingSummaryMessage("", platform, files, outputPath) + return body +} + // SendRecordingSummary 录制结束后发送录制文件摘要通知 // outputPath 为录制输出路径，用于获取剩余磁盘空间 func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform string, files []RecordingFileDetail, outputPath string) { @@ -303,34 +351,33 @@ func SendPipelineRecordingSummary( return } - // 分离已上传文件和保留文件 + // 分离已上传文件和本地保留文件 + var uploaded []RecordingFileDetail var kept []RecordingFileDetail - allUploaded := len(finalFiles) > 0 for _, f := range finalFiles { if f.Uploaded { - // 已上传文件不显示在最终列表中（已从磁盘删除） + uploaded = append(uploaded, f) } else { - allUploaded = false kept = append(kept, f) } } var title, body string - if allUploaded { - // 所有文件已上传并删除：显示实际上传的文件列表 - displayFiles := finalFiles - if len(displayFiles) == 0 { - // 回退：finalFiles 为空时用 originalFiles - displayFiles = originalFiles - } - if len(displayFiles) == 0 { - return - } + if len(uploaded) > 0 && len(kept) == 0 { + // 所有文件已上传：显示上传文件列表 title = fmt.Sprintf("%s 录制完成", hostName) - body = buildUploadedSummaryBody(platform, displayFiles) + body = buildUploadedSummaryBody(platform, uploaded) + } else if len(uploaded) > 0 && len(kept) > 0 { + // 部分上传、部分保留：显示两段 + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildMixedSummaryBody(platform, uploaded, kept, outputPath) } else if len(kept) > 0 { - // 有文件保留在磁盘：显示保留的文件列表 + // 全部本地保留 title, body = buildRecordingSummaryMessage(hostName, platform, kept, outputPath) + } else if len(originalFiles) > 0 { + // finalFiles 为空，回退到 originalFiles + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildRecordingSummaryMessageBody(platform, originalFiles, outputPath) } else { return } From 7ed283cf144f2c86b88cf0a8d1357051432d8e38 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:31:39 +0800 Subject: [PATCH 36/52] =?UTF-8?q?fix(pipeline):=20burn=5Fdelete=5Fass=20+?= =?UTF-8?q?=20upload=5Fsubtitles=20=E6=97=B6=20.ass=20=E6=9C=AA=E8=A2=AB?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit burn 阶段将 .ass 标记为 Deletable 后，cloud upload 的上传循环跳过它， 导致 .ass 未上传但通知显示「已上传到云端」。 修复： - 上传循环放行 Deletable 的 .ass（当 uploadSubtitles 时） - 上传后标记 Metadata[uploaded]，确保 executor 正确删除 修复后行为： - burn_delete_ass=true + upload_subtitles=true：.ass 先上传再删除 - burn_delete_ass=true + upload_subtitles=false：.ass 不上传，直接删除 - burn_delete_ass=false + upload_subtitles=true：.ass 正常上传 Co-Authored-By: Kscc --- src/pipeline/stages/extract_cover.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index 68fbb59..ea354d9 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -229,9 +229,12 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli for _, file := range input { // 跳过标记为待删除的中间文件（由其他阶段产出，不应上传） + // 例外：uploadSubtitles 时放行 Deletable 的 .ass（如 burn_delete_ass 标记的字幕） if file.Deletable { - output = append(output, file) - continue + if !s.uploadSubtitles || strings.ToLower(filepath.Ext(file.Path)) != ".ass" { + output = append(output, file) + continue + } } // 文件类型过滤（uploadSubtitles 时放行 .ass 字幕文件） @@ -332,6 +335,18 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } } + // 为已上传的 Deletable 文件标记 Metadata["uploaded"]（如 burn_delete_ass 标记的 .ass） + // 这些文件在上传循环中被放行（uploadSubtitles），但 Deletable 跳过逻辑未标记 uploaded + for i, f := range output { + if uploadedIndices[i] && f.Deletable { + if f.Metadata == nil { + f.Metadata = map[string]any{} + } + f.Metadata["uploaded"] = true + output[i] = f + } + } + // 全部上传成功后，根据配置标记文件为可删除（由 Executor 在管道全部成功后统一删除） // 任意一个上传失败，则不标记任何文件，确保本地文件全部保留 // 注意：不使用 Deletable 标记，避免后续阶段（如 CustomCommand）误跳过最终成品 From 15c973411f9937c02b705281bbf58d6d8ae0f2e0 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:54:12 +0800 Subject: [PATCH 37/52] =?UTF-8?q?fix(webapp):=20=E9=A2=84=E8=A7=88?= =?UTF-8?q?=E5=85=9C=E5=BA=95=E9=80=BB=E8=BE=91=E6=9C=AA=E6=AD=A3=E7=A1=AE?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E5=B7=B2=E5=88=A0=E9=99=A4=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=20.ass=20=E8=AF=AF=E6=98=BE=E4=B8=BA?= =?UTF-8?q?=E4=BF=9D=E7=95=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 兜底逻辑用 status !== 'deleted' && status !== 'uploaded_deleted' 判断视频 是否存在，但 deleteAfter 标记的视频状态是 'uploaded_delete'（无 d）， 导致兜底逻辑认为视频仍存在，.ass 不被清理。 修复：改为只认 'kept' 和 'uploaded'（本地保留）的视频为存在， 其他状态（deleted、uploaded_delete、uploaded_deleted）均视为不存在。 Co-Authored-By: Kscc --- src/webapp/src/component/config-info/CloudUploadSettings.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 582791c..90c44cb 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -232,7 +232,9 @@ const analyzeConfig = (config: any) => { const hasVideo = Object.entries(files).some(([name, f]) => { if (name === 'video.ass') return false; const ext = f.ext.toLowerCase(); - return (ext === '.flv' || ext === '.mp4' || ext === '.mkv') && f.status !== 'deleted' && f.status !== 'uploaded_deleted'; + if (ext !== '.flv' && ext !== '.mp4' && ext !== '.mkv') return false; + // 只有 kept 或 uploaded（本地保留）的视频才算"存在" + return f.status === 'kept' || f.status === 'uploaded'; }); if (!hasVideo) { files['video.ass'].status = 'deleted'; From f6002f088bbcf6537730212f91bc674d493087cf Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:59:24 +0800 Subject: [PATCH 38/52] =?UTF-8?q?fix(pipeline):=20=E5=8F=91=E7=8E=B0?= =?UTF-8?q?=E7=9A=84=20.ass=20=E6=96=87=E4=BB=B6=E4=B8=8D=E5=BA=94?= =?UTF-8?q?=E6=A0=87=E8=AE=B0=E4=B8=BA=E5=B7=B2=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deleteAllAfter 和 deleteAfter 路径中，从磁盘发现的关联 .ass 文件被标记 Metadata['uploaded'] = true，但实际未上传到云存储。通知错误将其报告为 「已上传到云端」。 修复：发现的 .ass 仅标记 Deletable（由 executor 删除），不标记 uploaded。 已通过 uploadSubtitles 上传的 .ass 不受影响（在上传循环中正确标记）。 Co-Authored-By: Kscc --- src/pipeline/stages/extract_cover.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index ea354d9..c647266 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -377,11 +377,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } if !alreadyInOutput { output = append(output, pipeline.FileInfo{ - Path: assPath, - Type: pipeline.FileTypeOther, - Metadata: map[string]any{ - "uploaded": true, // 全部上传成功，字幕文件将一并删除，标记为已上传 - }, + Path: assPath, + Type: pipeline.FileTypeOther, + Deletable: true, // 标记为可删除，由 executor 统一清理 }) assAdded[assPath] = true ctx.Logger.Infof("delete_all 模式：关联字幕文件 %s", assPath) @@ -421,8 +419,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli count++ } } - // 已上传的视频文件会被删除，其关联的 .ass 字幕文件也应标记为已上传 + // 已上传的视频文件会被删除，其关联的 .ass 字幕文件也应标记为可删除 // 避免摘要将已删除的字幕文件误显为本地保留文件 + // 注意：这些 .ass 未被上传到云存储，不标记 uploaded，仅标记 Deletable for i, f := range output { if !uploadedIndices[i] { continue // 仅处理已上传成功的视频文件 @@ -442,11 +441,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } if !alreadyInOutput { output = append(output, pipeline.FileInfo{ - Path: assPath, - Type: pipeline.FileTypeOther, - Metadata: map[string]any{ - "uploaded": true, - }, + Path: assPath, + Type: pipeline.FileTypeOther, + Deletable: true, // 标记为可删除，由 executor 统一清理 }) ctx.Logger.Infof("delete_uploaded 模式：关联字幕文件 %s", assPath) } From 54678eb83ffec90599d79238575c4cd2caffddbb Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:01:53 +0800 Subject: [PATCH 39/52] =?UTF-8?q?fix(recorders):=20=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E6=97=B6=E6=B8=85=E7=90=86=E6=97=A0=E5=85=B3?= =?UTF-8?q?=E8=81=94=E8=A7=86=E9=A2=91=E7=9A=84=E5=AD=A4=E7=AB=8B=20.ass?= =?UTF-8?q?=20=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DAA 发现的 .ass 标记为 Deletable（不标记 uploaded），executor 删除视频后 .ass 从 recordedFiles 加入 merged，但通知误显为本地保留。 修复：合并后检查 .ass 的关联视频是否在列表中，无视频的 .ass 被移除。 与 executor 的 cleanupOrphanedAssFiles 逻辑一致。 Co-Authored-By: Kscc --- src/recorders/recorder.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index a056be7..257a8ed 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -1090,6 +1090,8 @@ func (r *recorder) sendAccumulatedSummary() { merged = append(merged, f) } } + // 清理无关联视频的 .ass 文件（DAA/executor 可能已删除视频但 .ass 从 recordedFiles 加入） + merged = filterOrphanedAssFromDetails(merged) } r.getLogger().Infof("推送 Pipeline 录制摘要：%d 个文件", len(merged)) hostName := "" @@ -1233,6 +1235,33 @@ func collectSourceNames(sourceNames map[string]bool, fileSets ...[]pipeline.File } } +// filterOrphanedAssFromDetails 从文件详情列表中移除无关联视频的 .ass 文件 +// 当 DAA/executor 删除了视频但 .ass 从 recordedFiles 加入 merged 时， +// 需要清理这些孤立的 .ass，避免通知误显为本地保留 +func filterOrphanedAssFromDetails(details []notify.RecordingFileDetail) []notify.RecordingFileDetail { + videoExts := map[string]bool{".flv": true, ".mp4": true, ".mkv": true, ".ts": true} + videoBases := map[string]bool{} + for _, d := range details { + ext := strings.ToLower(filepath.Ext(d.Name)) + if videoExts[ext] { + base := strings.TrimSuffix(d.Name, ext) + videoBases[base] = true + } + } + var result []notify.RecordingFileDetail + for _, d := range details { + ext := strings.ToLower(filepath.Ext(d.Name)) + if ext == ".ass" { + base := strings.TrimSuffix(d.Name, ".ass") + if !videoBases[base] { + continue // 无关联视频，跳过 + } + } + result = append(result, d) + } + return result +} + func (r *recorder) logStreamURLRetry(err error) { if configs.IsDebug() || !strings.EqualFold(r.Live.GetPlatformCNName(), "SOOP") { r.warnStreamURLRetry(err, "failed to get stream url, will retry after 5s...", 0) From 42f8f9b519b336c5ddbe59259c71ae7f43c27e47 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:09:13 +0800 Subject: [PATCH 40/52] =?UTF-8?q?fix(webapp):=20=E9=A2=84=E8=A7=88?= =?UTF-8?q?=E6=9C=AA=E5=8C=BA=E5=88=86=E4=B8=AD=E9=97=B4=E4=BA=A7=E7=89=A9?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E4=B8=8A=E4=BC=A0=E5=88=97=E8=A1=A8=E5=92=8C?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E5=88=97=E8=A1=A8=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 预览用 'deleted' 同时表示「被处理阶段替代的源文件」和「被删除的文件」， 导致：1) 中间产物（如转码后的 .mp4）误出现在上传列表，2) 最终视频误出现在 保留列表，3) 孤儿 .ass 清理逻辑误判视频存在。 修复：新增 'intermediate' 状态表示中间产物（被转码/烧录替代的源文件）。 - intermediate 不参与上传（仅最终产物上传） - intermediate 在预览中显示为「删除」 - 孤儿 .ass 检查正确识别 intermediate 视频为不存在 Co-Authored-By: Kscc --- .../config-info/CloudUploadSettings.tsx | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 90c44cb..126237b 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -125,7 +125,8 @@ const analyzeConfig = (config: any) => { const delAllAfter = cu.delete_all_after_upload ?? false; const uploadAss = cu.upload_subtitles ?? false; - // 文件状态：uploaded(上传), deleted(删除), kept(保留) + // 文件状态：uploaded(上传保留), uploaded_deleted(上传后删除), intermediate(中间产物,不上传), + // deleted(删除), kept(本地保留) const files: Record = {}; // 初始文件 @@ -144,15 +145,13 @@ const analyzeConfig = (config: any) => { if (convert) { files['video.mp4'] = { ext: '.mp4', desc: '转码后视频', status: 'kept' }; if (deleteFlv) { - // 如果已经被标记为 uploaded，改为 uploaded_deleted + // 源文件被转码替代，标记为中间产物（不上传，最终由 executor 删除） if (files['video.flv'].status === 'uploaded') { files['video.flv'].status = 'uploaded_deleted'; } else { - files['video.flv'].status = 'deleted'; + files['video.flv'].status = 'intermediate'; } } else { - // delete_flv_after_convert=false：源视频保留在 ConvertMp4Stage output 中 - // immediate 模式下已在 pipeline 开头被上传，after_process 模式下也会被上传 if (files['video.flv'].status === 'kept') { files['video.flv'].status = 'uploaded'; } @@ -163,16 +162,13 @@ const analyzeConfig = (config: any) => { if (burn) { files['video.mkv'] = { ext: '.mkv', desc: '烧录后视频', status: 'kept' }; if (burnDelSource) { - // 删除转码后的 mp4 或原始 flv const target = convert ? 'video.mp4' : 'video.flv'; if (files[target].status === 'uploaded') { files[target].status = 'uploaded_deleted'; } else { - files[target].status = 'deleted'; + files[target].status = 'intermediate'; } } else { - // burn_delete_source=false：源视频保留在 BurnSubtitlesStage output 中 - // immediate 模式下已在 pipeline 开头被上传，after_process 模式下也会被上传 const src = convert ? 'video.mp4' : 'video.flv'; if (files[src] && files[src].status === 'kept') { files[src].status = 'uploaded'; @@ -234,6 +230,7 @@ const analyzeConfig = (config: any) => { const ext = f.ext.toLowerCase(); if (ext !== '.flv' && ext !== '.mp4' && ext !== '.mkv') return false; // 只有 kept 或 uploaded（本地保留）的视频才算"存在" + // intermediate（中间产物）和 deleted/uploaded_deleted（已删除）不算 return f.status === 'kept' || f.status === 'uploaded'; }); if (!hasVideo) { @@ -254,8 +251,8 @@ const analyzeConfig = (config: any) => { } else if (f.status === 'uploaded_deleted') { uploaded.push(name); deleted.push(name); - } else if (f.status === 'deleted') { - deleted.push(name); + } else if (f.status === 'deleted' || f.status === 'intermediate') { + deleted.push(name); // intermediate 是中间产物，显示为删除 } else { kept.push(name); } From b54af8ad62d74f4b0650e0c9bef6b5894c8c7751 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:17:02 +0800 Subject: [PATCH 41/52] =?UTF-8?q?fix(webapp):=20immediate=20=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E4=B8=8B=E9=A2=84=E8=A7=88=E4=B8=8A=E4=BC=A0=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E5=92=8C=E4=B8=AD=E9=97=B4=E4=BA=A7=E7=89=A9=E6=A0=87?= =?UTF-8?q?=E8=AE=B0=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit immediate 模式下 cloud_upload 最先执行，上传原始 .flv 和 .ass， 后续处理阶段产出的 .mp4/.mkv 不会被上传。但预览逻辑把 .flv 标记为 intermediate（不上传），把 .mp4 标记为 uploaded，与实际完全相反。 修复：immediate 模式下，源文件用 uploaded_deleted（已上传后被处理阶段 替代删除），处理阶段产出的 .mp4/.mkv 保持 kept（未被上传）。 after_process 模式下 intermediate 逻辑不变。 Co-Authored-By: Kscc --- .../config-info/CloudUploadSettings.tsx | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 126237b..57fb610 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -145,10 +145,13 @@ const analyzeConfig = (config: any) => { if (convert) { files['video.mp4'] = { ext: '.mp4', desc: '转码后视频', status: 'kept' }; if (deleteFlv) { - // 源文件被转码替代，标记为中间产物（不上传，最终由 executor 删除） - if (files['video.flv'].status === 'uploaded') { + if (isImmediate && upload) { + // immediate 模式：.flv 已在 pipeline 开头被上传，转换后被标记为 Deletable 并删除 + files['video.flv'].status = 'uploaded_deleted'; + } else if (files['video.flv'].status === 'uploaded') { files['video.flv'].status = 'uploaded_deleted'; } else { + // after_process 模式：源文件是中间产物，不参与上传 files['video.flv'].status = 'intermediate'; } } else { @@ -163,22 +166,36 @@ const analyzeConfig = (config: any) => { files['video.mkv'] = { ext: '.mkv', desc: '烧录后视频', status: 'kept' }; if (burnDelSource) { const target = convert ? 'video.mp4' : 'video.flv'; - if (files[target].status === 'uploaded') { + if (isImmediate && upload && !convert) { + // immediate 模式（无转码）：.flv 已在 pipeline 开头被上传，烧录后被标记为 Deletable 并删除 + files[target].status = 'uploaded_deleted'; + } else if (files[target].status === 'uploaded') { files[target].status = 'uploaded_deleted'; } else { + // after_process 模式或 immediate+convert（.mp4 未被上传）：中间产物 files[target].status = 'intermediate'; } } else { const src = convert ? 'video.mp4' : 'video.flv'; - if (files[src] && files[src].status === 'kept') { + // after_process 模式：源视频保留在 BurnSubtitlesStage output 中，会被 cloud_upload 上传 + // immediate 模式：源视频在 cloud_upload 之后才创建/保留，不会被上传 + if (!isImmediate && files[src] && files[src].status === 'kept') { files[src].status = 'uploaded'; } } if (burnDelAss) { - files['video.ass'].status = 'deleted'; + if (isImmediate && upload) { + // immediate 模式：.ass 已在 pipeline 开头被上传，烧录后被标记为 Deletable 并删除 + files['video.ass'].status = 'uploaded_deleted'; + } else { + files['video.ass'].status = 'deleted'; + } } } + // immediate 模式下，convert 和 burn 产出的文件（.mp4, .mkv）不会被上传 + // 它们在 cloud_upload 之后才创建，应保持 'kept' 状态 + // 封面 if (cover) { files['cover.jpg'] = { ext: '.jpg', desc: '视频封面', status: 'kept' }; From 8740b943e03fe21b6b261bb3e2029b3c4300027c Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:38:00 +0800 Subject: [PATCH 42/52] =?UTF-8?q?fix(recorders,webapp):=20=E5=88=86?= =?UTF-8?q?=E6=AE=B5=E9=87=8D=E5=90=AF=E6=97=B6=E6=91=98=E8=A6=81=E5=9C=A8?= =?UTF-8?q?=20Pipeline=20=E5=AE=8C=E6=88=90=E5=89=8D=E5=8F=91=E5=87=BA?= =?UTF-8?q?=E5=8F=8A=E9=A2=84=E8=A7=88=20burnDelSource=20=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 分段重启时序问题：CloseForRestart 先调用 r.Close() 让 run() 退出， run() 的 defer 触发 sendAccumulatedSummary，此时 suppressSummary 可能还未设置（在 pipelineState 中需要锁），导致摘要在 Pipeline 完成前 发出（pipelineDetails 为空，回退到 recordedFiles，无上传标记）。 Pipeline 完成后回调发现 summarySent=true，跳过发送，上传详情丢失。 修复：新增 closeForRestart 原子标记，在 CloseForRestart 中先设置标记 再调用 Close()。run() 的 defer 检查此标记，跳过 sendAccumulatedSummary。 TransferPipelineState 完成后由 onAllTasksDone 回调发送完整摘要。 2. 预览 burnDelSource=false 在 immediate 模式下错误标记 .mp4 为 uploaded。 .mp4 在 immediate 模式下不会被上传（cloud_upload 最先执行），应保持 kept。 Co-Authored-By: Kscc --- src/recorders/recorder.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index 257a8ed..188f0c3 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -316,6 +316,10 @@ type recorder struct { currentFileLock sync.RWMutex currentFilePath string + // closeForRestart 标记当前 recorder 正在分段重启，run() 退出时不发送摘要。 + // 在 CloseForRestart 中设置（早于 Close），run() 的 defer 中检查。 + closeForRestart atomic.Bool + // pipelineState 封装 Pipeline 相关的共享状态 // 使用指针，分段重启时新旧 recorder 共享同一份状态 pipelineState *pipelineSharedState @@ -977,7 +981,11 @@ func (r *recorder) run(ctx context.Context) { r.pipelineState.mu.Lock() r.pipelineState.runExited = true r.pipelineState.mu.Unlock() - r.sendAccumulatedSummary() + // 分段重启时跳过摘要：CloseForRestart 已设置此标记， + // TransferPipelineState 完成后由回调发送含完整 Pipeline 详情的摘要 + if !r.closeForRestart.Load() { + r.sendAccumulatedSummary() + } }() const minRetryInterval = 5 * time.Second @@ -1394,6 +1402,9 @@ func (r *recorder) Close() { } func (r *recorder) CloseForRestart() []notify.RecordingFileDetail { + // 先设置重启标记，使 run() 的 defer 跳过 sendAccumulatedSummary， + // 避免在 TransferPipelineState 之前发送不含 Pipeline 详情的摘要 + r.closeForRestart.Store(true) r.pipelineState.mu.Lock() r.pipelineState.suppressSummary = true r.pipelineState.mu.Unlock() From 77ba76cfa3179d77aa6c3fe8e6c54881675f7558 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:48:57 +0800 Subject: [PATCH 43/52] =?UTF-8?q?fix(pipeline):=20LastStageFiles=20?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E8=A2=AB=20deleteMarkedFiles=20=E6=B8=85?= =?UTF-8?q?=E9=99=A4=E4=B8=8A=E4=BC=A0=E6=A0=87=E8=AE=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit executor 保存 LastStageFiles 快照后调用 deleteMarkedFiles，后者清除已删除 文件的 Metadata[uploaded]。由于快照与 files 共享底层数组，LastStageFiles 中的上传标记也被清除。extractUploadedDetails 读取时无 uploaded 标记， 通知回退到 r.recordedFiles（无上传标注）。 修复：保存快照前深拷贝 files slice，断开与 deleteMarkedFiles 的共享。 Co-Authored-By: Kscc --- src/pipeline/executor.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 793c04d..1f33335 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -177,7 +177,20 @@ func (e *Executor) Execute( return results, ctx.Ctx.Err() } // 保存最后阶段输出的快照（用于回调提取上传文件详情），再执行清理 - ctx.LastStageFiles = files + // 必须深拷贝 Metadata，因为 deleteMarkedFiles 会清除 Metadata["uploaded"]， + // FileInfo.Metadata 是 map（引用类型），浅拷贝会共享同一 map + snapshot := make([]FileInfo, len(files)) + for i, f := range files { + cp := f + if f.Metadata != nil { + cp.Metadata = make(map[string]any, len(f.Metadata)) + for k, v := range f.Metadata { + cp.Metadata[k] = v + } + } + snapshot[i] = cp + } + ctx.LastStageFiles = snapshot keptFiles := e.deleteMarkedFiles(files) if len(results) > 0 { results[len(results)-1].OutputFiles = keptFiles From 543b04e67dacfcfefe5b6cd30e6fd984cafd7458 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:04:55 +0800 Subject: [PATCH 44/52] =?UTF-8?q?fix(recorders):=20=E9=83=A8=E5=88=86?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=97=B6=E5=9B=9E=E8=B0=83=E4=B8=A2=E5=A4=B1?= =?UTF-8?q?=E5=B7=B2=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6=E8=AF=A6=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onPipelineTaskComplete 仅在 CurrentFiles 为空（全部删除）时从 LastStageFiles 提取 uploaded 详情。当部分文件被上传删除、部分保留时（如 immediate 模式）， CurrentFiles 非空走 else 分支，已上传文件从 CurrentFiles 消失，LastStageFiles 的上传标记被忽略。 修复：始终从 LastStageFiles 提取 uploaded 详情，再合并 CurrentFiles 中未被 上传标记覆盖的保留文件。回退逻辑仅在两者都为空时触发。 Co-Authored-By: Kscc --- src/recorders/recorder.go | 58 ++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index 188f0c3..c9b6f21 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -1136,6 +1136,8 @@ func (r *recorder) sendAccumulatedSummary() { // onPipelineTaskComplete Pipeline 任务完成回调 // 由 Pipeline Manager 在 executeTask 结束时调用 func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { + r.getLogger().Infof("Pipeline 回调触发：状态=%s, CurrentFiles=%d, LastStageFiles=%d, InitialFiles=%d", + task.Status, len(task.CurrentFiles), len(task.LastStageFiles), len(task.InitialFiles)) // 沿 redirect 链找到当前活跃的共享状态 // 支持连续重启 A→B→C 场景：A 的回调通过 resolveState 找到 C 的状态 ps := resolveState(r.pipelineState) @@ -1146,21 +1148,27 @@ func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { // 收集任务的最终文件详情 switch task.Status { case pipeline.PipelineStatusCompleted: - finalFiles := task.CurrentFiles - if len(finalFiles) == 0 { - // 所有文件已上传并删除：从 LastStageFiles 提取真正上传的文件 - uploadedDetails := extractUploadedDetails(task.LastStageFiles) - if len(uploadedDetails) > 0 { - ps.details = append(ps.details, uploadedDetails...) - } else { - // 回退：无上传标记时用原始文件 - for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { - d.Uploaded = true - ps.details = append(ps.details, d) - } + // 始终从 LastStageFiles 提取已上传的文件（含 Metadata["uploaded"] 标记） + uploadedDetails := extractUploadedDetails(task.LastStageFiles) + if len(uploadedDetails) > 0 { + ps.details = append(ps.details, uploadedDetails...) + } + // 合并本地保留的文件（CurrentFiles 中未被上传标记覆盖的文件） + uploadedNames := map[string]bool{} + for _, d := range uploadedDetails { + uploadedNames[d.Name] = true + } + for _, d := range pipeline.FileInfoToDetails(task.CurrentFiles) { + if !uploadedNames[d.Name] { + ps.details = append(ps.details, d) + } + } + // 回退：无上传标记且无保留文件时用 InitialFiles + if len(uploadedDetails) == 0 && len(task.CurrentFiles) == 0 { + for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { + d.Uploaded = true + ps.details = append(ps.details, d) } - } else { - ps.details = append(ps.details, pipeline.FileInfoToDetails(finalFiles)...) } case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled: // 失败/取消时优先用 CurrentFiles（最后成功阶段的输出），回退到 InitialFiles @@ -1215,18 +1223,24 @@ func extractUploadedDetails(files []pipeline.FileInfo) []notify.RecordingFileDet if f.Type == pipeline.FileTypeCover { continue } + uploaded := false if f.Metadata != nil { - if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { - if f.Size > 0 { - details = append(details, notify.RecordingFileDetail{ - Name: filepath.Base(f.Path), - Size: f.Size, - Uploaded: true, - }) - } + if u, ok := f.Metadata["uploaded"].(bool); ok && u { + uploaded = true } } + if uploaded && f.Size > 0 { + details = append(details, notify.RecordingFileDetail{ + Name: filepath.Base(f.Path), + Size: f.Size, + Uploaded: true, + }) + } } + logrus.WithFields(logrus.Fields{ + "input_count": len(files), + "uploaded_count": len(details), + }).Debug("extractUploadedDetails") return details } From 4fbbd52f32852d0274f25b3e591f276e5c14cbc3 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:23:49 +0800 Subject: [PATCH 45/52] =?UTF-8?q?fix(pipeline,recorders):=20cleanupOrphane?= =?UTF-8?q?dAssFiles=20=E6=9C=AA=E6=A3=80=E6=9F=A5=20.ass=20=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E5=B7=B2=E8=A2=AB=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deleteMarkedFiles 删除 .ass 后不加入 kept，但 cleanupOrphanedAssFiles 的 第二轮（处理 Pipeline 中的 .ass）只检查关联视频是否存在，不检查 .ass 本身。 当 .mp4 存在时，已删除的 .ass 被错误保留在 kept 中，CurrentFiles 多出一个 条目，导致 onPipelineTaskComplete 的 uploaded/kept 分类错误。 修复：第二轮先检查 os.Stat，.ass 已不存在则跳过（不加入 kept）。 Co-Authored-By: Kscc --- src/pipeline/executor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 1f33335..828a8ad 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -335,6 +335,10 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File for _, f := range keptFiles { ext := strings.ToLower(filepath.Ext(f.Path)) if ext == ".ass" { + // 先检查 .ass 是否还在磁盘上（可能已被 deleteMarkedFiles 删除） + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + continue // 已被删除，不加入 kept + } dir := filepath.Dir(f.Path) base := strings.TrimSuffix(filepath.Base(f.Path), ".ass") key := filepath.Join(dir, base) From e52df5b4f98c156060958f3b53517f5d4a106b00 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:28:18 +0800 Subject: [PATCH 46/52] =?UTF-8?q?fix(pipeline):=20immediate=20=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E4=B8=8B=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6=E6=9C=AA?= =?UTF-8?q?=E6=A0=87=E8=AE=B0=20Metadata[uploaded]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metadata[uploaded] 的设置在 if !uploadFailed && uploadTiming != 'immediate' 块内，immediate 模式下整个块被跳过，uploaded 文件从未被标记。 extractUploadedDetails 读取 LastStageFiles 时找不到 uploaded 标记， 通知回退到无上传标注的格式。 修复：在上传循环中上传成功时立即设置 Metadata[uploaded] = true， 不依赖后续删除逻辑块。同时修复 cleanupOrphanedAssFiles 未检查 .ass 是否已被 deleteMarkedFiles 删除的问题。 Co-Authored-By: Kscc --- src/pipeline/stages/extract_cover.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index c647266..45b6512 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -320,6 +320,11 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli s.logs += fmt.Sprintf("上传成功: %s -> %s/%s\n", fileName, s.storageName, targetPath) ctx.Logger.Infof("文件上传成功: %s -> %s/%s", fileName, s.storageName, targetPath) + // 标记已上传（立即设置，不依赖后续删除逻辑块，确保 immediate 模式下也有标记） + if file.Metadata == nil { + file.Metadata = map[string]any{} + } + file.Metadata["uploaded"] = true uploadedIndices[len(output)] = true // 记录上传成功的文件索引 output = append(output, file) From 5cc21c8afdd45d45920f8f210f3755947351d5c2 Mon Sep 17 00:00:00 2001 From: Joftal <42807405+Joftal@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:34:11 +0800 Subject: [PATCH 47/52] =?UTF-8?q?fix(pipeline):=20immediate=20=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E4=B8=8B=E5=90=8E=E7=BB=AD=E9=98=B6=E6=AE=B5=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E6=96=87=E4=BB=B6=E5=90=8E=E4=B8=8A=E4=BC=A0=E6=A0=87?= =?UTF-8?q?=E8=AE=B0=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit immediate 模式下 cloud_upload 最先执行，上传 .flv 后设置 Metadata[uploaded]。 后续 fix_flv 阶段创建新 FileInfo 替换原始 .flv，新对象无 Metadata， 上传标记丢失。.ass 不经过 fix_flv，标记保留。 修复： 1. 上传循环中上传成功时立即设置 Metadata[uploaded]（不依赖删除逻辑块） 2. PipelineContext 新增 UploadedPaths 记录已上传文件路径 3. 所有阶段执行完毕后，按基名匹配为被替换的文件恢复上传标记 Co-Authored-By: Kscc --- src/pipeline/executor.go | 49 ++++++++++++++++++++++++++++ src/pipeline/stages/extract_cover.go | 4 +++ src/pipeline/types.go | 5 +++ 3 files changed, 58 insertions(+) diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 828a8ad..3c3c716 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -67,6 +67,7 @@ func (e *Executor) Execute( files := initialFiles results := make([]StageResult, 0, len(config.Stages)) + ctx.UploadedPaths = make(map[string]bool) stageIndex := 0 for i, stageCfg := range config.Stages { @@ -167,6 +168,54 @@ func (e *Executor) Execute( }).Debug("stage completed") } + // 去重：多个阶段可能添加同名文件（如 cloud_upload 上传 .ass 后，burn 又添加 Deletable 副本） + // 保留第一个出现的（保留最早阶段的 Metadata，如 uploaded 标记） + seen := map[string]int{} + var deduped []FileInfo + for _, f := range files { + if idx, exists := seen[f.Path]; exists { + // 合并 Deletable 标记到已有条目 + if f.Deletable && !deduped[idx].Deletable { + deduped[idx].Deletable = true + } + continue + } + seen[f.Path] = len(deduped) + deduped = append(deduped, f) + } + files = deduped + + // 重新标记被后续阶段替换的已上传文件 + // immediate 模式下 cloud_upload 最先执行，后续阶段（fix_flv） + // 创建新 FileInfo 替换原始文件，丢失 Metadata["uploaded"]。 + // 通过 UploadedPaths 按基名+扩展名匹配，为同类型的替换文件恢复上传标记。 + // 不匹配不同扩展名的派生文件（如 .flv 上传后生成的 .mp4/.mkv 不应标记为 uploaded）。 + if len(ctx.UploadedPaths) > 0 { + for uploadedPath := range ctx.UploadedPaths { + uploadedBase := strings.TrimSuffix(filepath.Base(uploadedPath), filepath.Ext(uploadedPath)) + uploadedExt := strings.ToLower(filepath.Ext(uploadedPath)) + for i, f := range files { + if f.Metadata != nil { + if u, ok := f.Metadata["uploaded"].(bool); ok && u { + continue // 已有标记，跳过 + } + } + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != uploadedExt { + continue // 不同扩展名，不是直接替换 + } + fileBase := strings.TrimSuffix(filepath.Base(f.Path), ext) + if fileBase == uploadedBase { + if files[i].Metadata == nil { + files[i].Metadata = map[string]any{} + } + files[i].Metadata["uploaded"] = true + e.logger.Infof("恢复上传标记: %s (匹配 %s)", f.Path, filepath.Base(uploadedPath)) + } + } + } + } + // 全部成功：执行延迟删除，并让最终结果反映磁盘上的文件 // 如果上下文已取消，跳过删除以保留所有文件，返回取消错误让任务标记为 Cancelled if ctx.Ctx.Err() != nil { diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index 45b6512..1d54e29 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -325,6 +325,10 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli file.Metadata = map[string]any{} } file.Metadata["uploaded"] = true + // 记录上传路径，用于后续阶段替换文件后重新标记 + if ctx.UploadedPaths != nil { + ctx.UploadedPaths[file.Path] = true + } uploadedIndices[len(output)] = true // 记录上传成功的文件索引 output = append(output, file) diff --git a/src/pipeline/types.go b/src/pipeline/types.go index 28792d9..70e569d 100644 --- a/src/pipeline/types.go +++ b/src/pipeline/types.go @@ -97,6 +97,11 @@ type PipelineContext struct { // LastStageFiles 最后阶段输出的文件快照（清理前保存） // 用于回调提取上传文件详情，因为 deleteMarkedFiles 会删除文件 LastStageFiles []FileInfo + + // UploadedPaths 记录 cloud_upload 阶段成功上传的文件路径 + // 用于在所有阶段执行完毕后，为被后续阶段替换的文件重新标记 Metadata["uploaded"] + // （如 fix_flv 替换 .flv 后，新 FileInfo 丢失上传标记） + UploadedPaths map[string]bool } // Stage 管道阶段接口 From 524f3329c7173047757add50472c8858b5a5a5ca Mon Sep 17 00:00:00 2001 From: kira1928 Date: Mon, 17 Aug 2026 22:51:54 +0900 Subject: [PATCH 48/52] =?UTF-8?q?fix(test):=20=E4=BF=AE=E5=A4=8D=20E2E=20a?= =?UTF-8?q?ctive=5Frecordings=20=E6=B3=84=E6=BC=8F=E5=B9=B6=E7=BB=88?= =?UTF-8?q?=E6=AD=A2=E6=B8=B2=E6=9F=93=E5=A4=B1=E8=B4=A5=E7=9A=84=E5=BD=95?= =?UTF-8?q?=E5=88=B6=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E2E 中 update-full/update-workflow 的 "active_recordings 为 0" 断言偶发失败，根因有三： 1. fixture 模板 out_put_tmpl 使用了 live.Info 上不存在的字段 {{.StartTime}}， 导致每次文件名渲染都失败（自 #1061 起）。改为合法的时间戳写法。 2. recording.spec.ts 通过 UI 添加的 dev 直播间从不清理；workers=1 下所有 spec 共用同一后端进程，残留房间被 mock 持续判定为在播，泄漏进后续 spec 的断言。 增加 afterAll 统一删除全部直播间。 3. 文件名渲染失败后 recorder 会以 5s 间隔无限重试、刷屏且被计入活跃录制数。 现改为派发一次 LiveEnd 交由 manager 回收（与 stopRetryForExplicitOffline 同构），避免无意义的重试风暴。 并将两处 active_recordings 断言改为轮询，容忍录制器回收的短暂延迟。 via [HAPI](https://hapi.run) Co-Authored-By: HAPI Co-Authored-By: Claude Opus 4.8 --- src/recorders/recorder.go | 7 ++++- src/recorders/recorder_test.go | 11 ++++++-- .../fixtures/ffmpeg-test-config.template.yml | 2 +- tests/e2e/fixtures/test-config.template.yml | 2 +- tests/e2e/recording.spec.ts | 28 +++++++++++++++++-- tests/e2e/update-full.spec.ts | 10 +++++-- tests/e2e/update-workflow.spec.ts | 10 +++++-- 7 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index dfc9753..f031319 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -381,7 +381,12 @@ func (r *recorder) tryRecord(ctx context.Context) { buf := new(bytes.Buffer) if err = tmpl.Execute(buf, info); err != nil { - r.getLogger().WithError(err).Error("failed to render filename, recording aborted") + // 文件名模板渲染失败通常是配置错误（如引用了不存在的字段），每次重试都会 + // 重复失败。派发一次 LiveEnd 让 manager 回收该 recorder，避免无意义的 5s + // 重试风暴与日志刷屏，也不再被误计入活跃录制数。listener 的内部状态不受 + // 影响，房间真正下播再开播时仍会以修正后的配置重新创建 recorder。 + r.getLogger().WithError(err).Error("failed to render filename, stopping recorder (check out_put_tmpl)") + r.ed.DispatchEvent(events.NewEvent(listeners.LiveEnd, r.Live)) return } // 使用层级配置的 OutPutPath diff --git a/src/recorders/recorder_test.go b/src/recorders/recorder_test.go index f0899c9..4f85450 100644 --- a/src/recorders/recorder_test.go +++ b/src/recorders/recorder_test.go @@ -11,8 +11,11 @@ import ( gomock "go.uber.org/mock/gomock" "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/listeners" "github.com/bililive-go/bililive-go/src/live" livemock "github.com/bililive-go/bililive-go/src/live/mock" + "github.com/bililive-go/bililive-go/src/pkg/events" + eventsmock "github.com/bililive-go/bililive-go/src/pkg/events/mock" "github.com/bililive-go/bililive-go/src/pkg/livelogger" ) @@ -34,16 +37,20 @@ func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) { l.EXPECT().GetStreamInfos().Return([]*live.StreamUrlInfo{{Url: streamURL}}, nil) l.EXPECT().GetLogger().Return(logger) + // 渲染失败时 recorder 应派发一次 LiveEnd 交由 manager 回收，而不是无限重试。 + ed := eventsmock.NewMockDispatcher(ctrl) + ed.EXPECT().DispatchEvent(events.NewEvent(listeners.LiveEnd, l)) + cache := gcache.New(1).LRU().Build() if err := cache.Set(l, &live.Info{Live: l}); err != nil { t.Fatalf("写入直播信息缓存失败: %v", err) } - r := &recorder{Live: l, cache: cache} + r := &recorder{Live: l, cache: cache, ed: ed} r.tryRecord(context.Background()) logs := logger.GetLogs() - if !strings.Contains(logs, "failed to render filename, recording aborted") { + if !strings.Contains(logs, "failed to render filename, stopping recorder") { t.Fatalf("未记录文件名渲染失败日志: %s", logs) } if !strings.Contains(logs, "render failed") { diff --git a/tests/e2e/fixtures/ffmpeg-test-config.template.yml b/tests/e2e/fixtures/ffmpeg-test-config.template.yml index cb79c3f..cb464ea 100644 --- a/tests/e2e/fixtures/ffmpeg-test-config.template.yml +++ b/tests/e2e/fixtures/ffmpeg-test-config.template.yml @@ -16,7 +16,7 @@ log: rotate_days: 7 feature: remove_symbol_other_character: false -out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{.StartTime}}' +out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}' video_split_strategies: on_room_name_changed: false max_duration: 0s diff --git a/tests/e2e/fixtures/test-config.template.yml b/tests/e2e/fixtures/test-config.template.yml index 54f6268..fc3197c 100644 --- a/tests/e2e/fixtures/test-config.template.yml +++ b/tests/e2e/fixtures/test-config.template.yml @@ -18,7 +18,7 @@ feature: # '{{ .Live.GetPlatformCNName }}/{{ .HostName | filenameFilter }}/[{{ now | date "2006-01-02 15-04-05"}}][{{ .HostName | filenameFilter }}][{{ .RoomName | filenameFilter }}].flv' # ./平台名称/主播名字/[时间戳][主播名字][房间名字].flv # https://github.com/bililive-go/bililive-go/wiki/More-Tips -out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{.StartTime}}' +out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}' video_split_strategies: on_room_name_changed: false max_duration: 0s diff --git a/tests/e2e/recording.spec.ts b/tests/e2e/recording.spec.ts index 77c1781..0f8d2e8 100644 --- a/tests/e2e/recording.spec.ts +++ b/tests/e2e/recording.spec.ts @@ -1,8 +1,8 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, request as apiRequest } from '@playwright/test'; /** * 直播录制功能测试 - * + * * 使用 osrp-stream-tester 提供的 dev 直播间进行测试 * 测试添加直播间、开始/停止录制等功能 */ @@ -10,6 +10,30 @@ import { test, expect } from '@playwright/test'; // dev 测试流服务器地址 const DEV_STREAM_SERVER = 'http://127.0.0.1:8888'; const DEV_STREAM_URL = `${DEV_STREAM_SERVER}/live/test.flv`; +// bililive-go 后端地址 +const BGO_SERVER = 'http://127.0.0.1:8080'; + +// 本文件会通过 UI 添加 dev 直播间；由于所有 spec 共用同一个后端进程（workers=1）， +// 若不清理，残留的监控房间会一直被判定为"在播"，导致后续 spec（如 update-*）的 +// active_recordings 断言失败。这里在文件所有用例结束后统一删除所有直播间，保证隔离。 +test.afterAll(async () => { + const ctx = await apiRequest.newContext(); + try { + const res = await ctx.get(`${BGO_SERVER}/api/lives`); + if (res.ok()) { + const lives = await res.json(); + if (Array.isArray(lives)) { + for (const l of lives) { + if (l?.id) { + await ctx.delete(`${BGO_SERVER}/api/lives/${l.id}`); + } + } + } + } + } finally { + await ctx.dispose(); + } +}); test.describe('直播间管理测试', () => { test.beforeEach(async ({ page }) => { diff --git a/tests/e2e/update-full.spec.ts b/tests/e2e/update-full.spec.ts index ea22386..c949175 100644 --- a/tests/e2e/update-full.spec.ts +++ b/tests/e2e/update-full.spec.ts @@ -208,9 +208,13 @@ test.describe('优雅更新逻辑测试', () => { const response = await request.get('/api/update/launcher'); expect(response.ok()).toBeTruthy(); - const data = await response.json(); - // 测试环境应该没有活跃录制 - expect(data.active_recordings).toBe(0); + // 测试环境应该没有活跃录制。前序 spec 清理房间后录制器回收存在短暂延迟， + // 使用轮询等待其归零，避免时序相关的偶发失败。 + await expect.poll(async () => { + const res = await request.get('/api/update/launcher'); + const d = await res.json(); + return d.active_recordings; + }, { timeout: 10000 }).toBe(0); }); test('更新状态不影响其他 API', async ({ request }) => { diff --git a/tests/e2e/update-workflow.spec.ts b/tests/e2e/update-workflow.spec.ts index 471a713..7f7785f 100644 --- a/tests/e2e/update-workflow.spec.ts +++ b/tests/e2e/update-workflow.spec.ts @@ -254,9 +254,13 @@ test.describe('优雅更新逻辑测试', () => { const response = await request.get('/api/update/launcher'); expect(response.ok()).toBeTruthy(); - const data = await response.json(); - // 测试环境应该没有活跃录制 - expect(data.active_recordings).toBe(0); + // 测试环境应该没有活跃录制。前序 spec 清理房间后录制器回收存在短暂延迟， + // 使用轮询等待其归零，避免时序相关的偶发失败。 + await expect.poll(async () => { + const res = await request.get('/api/update/launcher'); + const d = await res.json(); + return d.active_recordings; + }, { timeout: 10000 }).toBe(0); }); test('更新状态不影响其他核心 API', async ({ request }) => { From ad81d2b9cfef8033eecab9ce1f35b5697935e82d Mon Sep 17 00:00:00 2001 From: kira1928 Date: Tue, 18 Aug 2026 01:10:20 +0900 Subject: [PATCH 49/52] =?UTF-8?q?fix:=20=E6=A0=B9=E6=8D=AE=20review=20?= =?UTF-8?q?=E6=94=B6=E7=B4=A7=E5=BD=95=E5=88=B6=E5=99=A8=E5=81=9C=E6=AD=A2?= =?UTF-8?q?=E4=B8=8E=20E2E=20=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/recorders/event.go | 7 +-- src/recorders/manager.go | 3 ++ src/recorders/manager_test.go | 22 +++++++++ src/recorders/recorder.go | 8 ++-- src/recorders/recorder_test.go | 5 +- .../fixtures/ffmpeg-test-config.template.yml | 2 +- tests/e2e/fixtures/test-config.template.yml | 2 +- tests/e2e/recording.spec.ts | 47 ++++++++++++++++--- 8 files changed, 78 insertions(+), 18 deletions(-) diff --git a/src/recorders/event.go b/src/recorders/event.go index 346b953..afd9dcb 100644 --- a/src/recorders/event.go +++ b/src/recorders/event.go @@ -3,7 +3,8 @@ package recorders import "github.com/bililive-go/bililive-go/src/pkg/events" const ( - RecorderStart events.EventType = "RecorderStart" - RecorderStop events.EventType = "RecorderStop" - RecorderRestart events.EventType = "RecorderRestart" + RecorderStart events.EventType = "RecorderStart" + RecorderStop events.EventType = "RecorderStop" + RecorderRestart events.EventType = "RecorderRestart" + RecorderStopRequested events.EventType = "RecorderStopRequested" ) diff --git a/src/recorders/manager.go b/src/recorders/manager.go index 6925772..6138716 100644 --- a/src/recorders/manager.go +++ b/src/recorders/manager.go @@ -133,6 +133,9 @@ func (m *manager) registryListener(ctx context.Context, ed events.Dispatcher) { }) ed.AddEventListener(listeners.LiveEnd, removeEvtListener) ed.AddEventListener(listeners.ListenStop, removeEvtListener) + // RecorderStopRequested 仅表示 recorder 自身遇到不可重试错误，需要被回收； + // 它不具有 LiveEnd 的全局直播状态语义，避免错误结束仍在进行的直播会话。 + ed.AddEventListener(RecorderStopRequested, removeEvtListener) } func (m *manager) Start(ctx context.Context) error { diff --git a/src/recorders/manager_test.go b/src/recorders/manager_test.go index f292349..87ee6f5 100644 --- a/src/recorders/manager_test.go +++ b/src/recorders/manager_test.go @@ -12,6 +12,7 @@ import ( "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" livemock "github.com/bililive-go/bililive-go/src/live/mock" + "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" "github.com/bililive-go/bililive-go/src/types" ) @@ -56,6 +57,27 @@ func TestManagerAddAndRemoveRecorder(t *testing.T) { assert.False(t, m.HasRecorder(context.Background(), "test")) } +// TestRecorderStopRequestedRemovesOnlyRecorder 验证 recorder 的内部停止请求会被 +// manager 消费并回收 recorder，无需伪造具有全局直播状态语义的 LiveEnd。 +func TestRecorderStopRequestedRemovesOnlyRecorder(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ctx := context.Background() + ed := events.NewDispatcher(ctx) + l := livemock.NewMockLive(ctrl) + l.EXPECT().GetLiveId().Return(types.LiveID("test")).AnyTimes() + + r := NewMockRecorder(ctrl) + r.EXPECT().Close() + m := &manager{savers: map[types.LiveID]Recorder{"test": r}} + m.registryListener(ctx, ed) + + ed.DispatchEventSync(events.NewEvent(RecorderStopRequested, l)) + + assert.False(t, m.HasRecorder(ctx, "test")) +} + // TestRestartRecorderRaceWithLiveEnd 验证 RestartRecorder 和 LiveEnd（RemoveRecorder） // 并发执行时不会产生僵尸录制器。 // diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index f031319..ebb383e 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -382,11 +382,11 @@ func (r *recorder) tryRecord(ctx context.Context) { buf := new(bytes.Buffer) if err = tmpl.Execute(buf, info); err != nil { // 文件名模板渲染失败通常是配置错误（如引用了不存在的字段），每次重试都会 - // 重复失败。派发一次 LiveEnd 让 manager 回收该 recorder，避免无意义的 5s - // 重试风暴与日志刷屏，也不再被误计入活跃录制数。listener 的内部状态不受 - // 影响，房间真正下播再开播时仍会以修正后的配置重新创建 recorder。 + // 重复失败。请求 manager 仅回收该 recorder，避免无意义的 5s 重试风暴与 + // 日志刷屏，也不再被误计入活跃录制数；不要派发具有全局直播状态语义的 + // LiveEnd，否则会错误地提前结束仍在进行的 live session。 r.getLogger().WithError(err).Error("failed to render filename, stopping recorder (check out_put_tmpl)") - r.ed.DispatchEvent(events.NewEvent(listeners.LiveEnd, r.Live)) + r.ed.DispatchEvent(events.NewEvent(RecorderStopRequested, r.Live)) return } // 使用层级配置的 OutPutPath diff --git a/src/recorders/recorder_test.go b/src/recorders/recorder_test.go index 4f85450..b41d29f 100644 --- a/src/recorders/recorder_test.go +++ b/src/recorders/recorder_test.go @@ -11,7 +11,6 @@ import ( gomock "go.uber.org/mock/gomock" "github.com/bililive-go/bililive-go/src/configs" - "github.com/bililive-go/bililive-go/src/listeners" "github.com/bililive-go/bililive-go/src/live" livemock "github.com/bililive-go/bililive-go/src/live/mock" "github.com/bililive-go/bililive-go/src/pkg/events" @@ -37,9 +36,9 @@ func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) { l.EXPECT().GetStreamInfos().Return([]*live.StreamUrlInfo{{Url: streamURL}}, nil) l.EXPECT().GetLogger().Return(logger) - // 渲染失败时 recorder 应派发一次 LiveEnd 交由 manager 回收，而不是无限重试。 + // 渲染失败时 recorder 应请求 manager 仅回收自身，而不是错误派发 LiveEnd。 ed := eventsmock.NewMockDispatcher(ctrl) - ed.EXPECT().DispatchEvent(events.NewEvent(listeners.LiveEnd, l)) + ed.EXPECT().DispatchEvent(events.NewEvent(RecorderStopRequested, l)) cache := gcache.New(1).LRU().Build() if err := cache.Set(l, &live.Info{Live: l}); err != nil { diff --git a/tests/e2e/fixtures/ffmpeg-test-config.template.yml b/tests/e2e/fixtures/ffmpeg-test-config.template.yml index cb464ea..9b67e55 100644 --- a/tests/e2e/fixtures/ffmpeg-test-config.template.yml +++ b/tests/e2e/fixtures/ffmpeg-test-config.template.yml @@ -16,7 +16,7 @@ log: rotate_days: 7 feature: remove_symbol_other_character: false -out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}' +out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}.flv' video_split_strategies: on_room_name_changed: false max_duration: 0s diff --git a/tests/e2e/fixtures/test-config.template.yml b/tests/e2e/fixtures/test-config.template.yml index fc3197c..9320baf 100644 --- a/tests/e2e/fixtures/test-config.template.yml +++ b/tests/e2e/fixtures/test-config.template.yml @@ -18,7 +18,7 @@ feature: # '{{ .Live.GetPlatformCNName }}/{{ .HostName | filenameFilter }}/[{{ now | date "2006-01-02 15-04-05"}}][{{ .HostName | filenameFilter }}][{{ .RoomName | filenameFilter }}].flv' # ./平台名称/主播名字/[时间戳][主播名字][房间名字].flv # https://github.com/bililive-go/bililive-go/wiki/More-Tips -out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}' +out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}.flv' video_split_strategies: on_room_name_changed: false max_duration: 0s diff --git a/tests/e2e/recording.spec.ts b/tests/e2e/recording.spec.ts index 0f8d2e8..4308cac 100644 --- a/tests/e2e/recording.spec.ts +++ b/tests/e2e/recording.spec.ts @@ -13,19 +13,55 @@ const DEV_STREAM_URL = `${DEV_STREAM_SERVER}/live/test.flv`; // bililive-go 后端地址 const BGO_SERVER = 'http://127.0.0.1:8080'; +interface LiveInfo { + id: string; + live_url?: string; +} + +const preExistingLiveIds = new Set(); +let baselineCaptured = false; + +// 本地运行时 Playwright 可能复用开发者已经启动的 bililive-go。先记录既有房间， +// 避免测试清理误删开发者的持久化配置。 +test.beforeAll(async () => { + const ctx = await apiRequest.newContext(); + try { + const res = await ctx.get(`${BGO_SERVER}/api/lives`); + if (!res.ok()) { + return; + } + const lives: LiveInfo[] = await res.json(); + if (!Array.isArray(lives)) { + return; + } + for (const live of lives) { + if (live.id) { + preExistingLiveIds.add(live.id); + } + } + baselineCaptured = true; + } finally { + await ctx.dispose(); + } +}); + // 本文件会通过 UI 添加 dev 直播间；由于所有 spec 共用同一个后端进程（workers=1）， // 若不清理，残留的监控房间会一直被判定为"在播"，导致后续 spec（如 update-*）的 -// active_recordings 断言失败。这里在文件所有用例结束后统一删除所有直播间，保证隔离。 +// active_recordings 断言失败。这里只删除本文件新建的 dev 直播间，既保证测试隔离， +// 也不会在复用本地服务时误删开发者已有的监控房间。 test.afterAll(async () => { + if (!baselineCaptured) { + return; + } const ctx = await apiRequest.newContext(); try { const res = await ctx.get(`${BGO_SERVER}/api/lives`); if (res.ok()) { - const lives = await res.json(); + const lives: LiveInfo[] = await res.json(); if (Array.isArray(lives)) { - for (const l of lives) { - if (l?.id) { - await ctx.delete(`${BGO_SERVER}/api/lives/${l.id}`); + for (const live of lives) { + if (live.id && live.live_url === DEV_STREAM_URL && !preExistingLiveIds.has(live.id)) { + await ctx.delete(`${BGO_SERVER}/api/lives/${live.id}`); } } } @@ -248,4 +284,3 @@ test.describe('紧急情况：停止所有录制', () => { expect(count).toBeGreaterThanOrEqual(0); }); }); - From 7dcee5132b06243e6c87830a94f33623f390f25e Mon Sep 17 00:00:00 2001 From: kira1928 Date: Sun, 23 Aug 2026 01:46:57 +0900 Subject: [PATCH 50/52] =?UTF-8?q?fix:=20=E6=9A=82=E6=97=B6=E5=85=B3?= =?UTF-8?q?=E9=97=AD=E5=B9=B3=E5=8F=B0=E7=BA=A7=E8=AE=BF=E9=97=AE=E9=99=90?= =?UTF-8?q?=E6=B5=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/configs/config_test.go | 10 +++---- src/pkg/ratelimit/ratelimit.go | 44 +++++++++++++++++++++++++++-- src/pkg/ratelimit/ratelimit_test.go | 27 ++++++++++++++++-- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/src/configs/config_test.go b/src/configs/config_test.go index eb857ec..555832d 100644 --- a/src/configs/config_test.go +++ b/src/configs/config_test.go @@ -97,7 +97,7 @@ func TestGetPlatformMinAccessInterval(t *testing.T) { assert.Equal(t, 1, interval) // 默认最小间隔为 1 秒，防止无限制高频访问 } -func TestDefaultPlatformRateLimitSurvivesTransientConfigUpdate(t *testing.T) { +func TestPlatformRateLimitRemainsDisabledAfterConfigUpdate(t *testing.T) { const roomURL = "https://live.douyin.com/123456" limiter := ratelimit.GetGlobalRateLimiter() t.Cleanup(func() { @@ -109,15 +109,15 @@ func TestDefaultPlatformRateLimitSurvivesTransientConfigUpdate(t *testing.T) { cfg.RefreshLiveRoomIndexCache() SetCurrentConfig(cfg) - if got := limiter.GetAllPlatformLimits()[PlatformKeyDouyin]; got != 1 { - t.Fatalf("初始默认平台限流 = %d 秒，期望 1 秒", got) + if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { + t.Fatalf("配置加载后平台限流应保持关闭，实际限制：%v", limits) } if _, err := SetLiveRoomId(roomURL, types.LiveID("douyin-test")); err != nil { t.Fatalf("写入临时 LiveId 失败: %v", err) } - if got := limiter.GetAllPlatformLimits()[PlatformKeyDouyin]; got != 1 { - t.Fatalf("临时配置更新后的默认平台限流 = %d 秒，期望仍为 1 秒", got) + if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { + t.Fatalf("临时配置更新后平台限流应保持关闭，实际限制：%v", limits) } } diff --git a/src/pkg/ratelimit/ratelimit.go b/src/pkg/ratelimit/ratelimit.go index fd2a73b..cd895cf 100644 --- a/src/pkg/ratelimit/ratelimit.go +++ b/src/pkg/ratelimit/ratelimit.go @@ -11,6 +11,7 @@ import ( type PlatformRateLimiter struct { limiters map[string]*PlatformLimiter // 平台名称 -> 限制器 mu sync.RWMutex // 读写锁保护map + enabled bool // 是否实际应用平台级访问限制 } // PlatformLimiter 单个平台的频率限制器 @@ -21,8 +22,17 @@ type PlatformLimiter struct { inFlight chan struct{} // 限制同一平台最多一个请求在途 } -var globalRateLimiter = &PlatformRateLimiter{ - limiters: make(map[string]*PlatformLimiter), +// 平台级访问限制暂时关闭，以避免同平台请求串行排队导致新直播间长时间无法开始录制。 +// 保留限制器实现和配置结构，后续改善调度策略后可重新启用。 +const globalRateLimitEnabled = false + +var globalRateLimiter = newPlatformRateLimiter(globalRateLimitEnabled) + +func newPlatformRateLimiter(enabled bool) *PlatformRateLimiter { + return &PlatformRateLimiter{ + limiters: make(map[string]*PlatformLimiter), + enabled: enabled, + } } // GetGlobalRateLimiter 获取全局速率限制器实例 @@ -32,6 +42,10 @@ func GetGlobalRateLimiter() *PlatformRateLimiter { // SetPlatformLimit 设置或更新指定平台的访问频率限制 func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec int) { + if !prl.enabled { + return + } + if intervalSec <= 0 { // 如果间隔为0或负数，移除该平台的限制 prl.mu.Lock() @@ -63,7 +77,7 @@ func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec in // EnsurePlatformLimit 在平台尚无限制时设置默认值，不覆盖已经显式配置的间隔。 // 用于房间对象早于配置持久化创建的路径，保证首次请求也不会绕过平台保护。 func (prl *PlatformRateLimiter) EnsurePlatformLimit(platform string, intervalSec int) { - if platform == "" || intervalSec <= 0 { + if !prl.enabled || platform == "" || intervalSec <= 0 { return } @@ -90,6 +104,10 @@ func (prl *PlatformRateLimiter) WaitForPlatform(platform string) { // 如果平台没有设置限制，立即返回 true // 返回 true 表示成功获取访问权限，false 表示被 context 取消 func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, platform string) bool { + if !prl.enabled { + return true + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -108,6 +126,10 @@ func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, // 仅限制请求开始间隔不足以保护启动阶段：当前一个请求耗时超过最小间隔时，后续请求仍会 // 重叠执行。这里把并发槽位与开始间隔合并为一次许可，使大量直播间并发初始化时仍按平台串行。 func (prl *PlatformRateLimiter) AcquirePlatformWithContext(ctx context.Context, platform string) (release func(), ok bool) { + if !prl.enabled { + return func() {}, true + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -172,6 +194,10 @@ func waitForLimiterWithContext(ctx context.Context, limiter *PlatformLimiter) bo // GetPlatformNextAllowedTime 获取平台下次允许访问的时间 func (prl *PlatformRateLimiter) GetPlatformNextAllowedTime(platform string) time.Time { + if !prl.enabled { + return time.Now() + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -197,6 +223,10 @@ func (prl *PlatformRateLimiter) RemovePlatformLimit(platform string) { // GetAllPlatformLimits 获取所有平台的当前限制设置 func (prl *PlatformRateLimiter) GetAllPlatformLimits() map[string]int { + if !prl.enabled { + return map[string]int{} + } + prl.mu.RLock() defer prl.mu.RUnlock() @@ -219,6 +249,10 @@ type WaitInfo struct { // GetPlatformWaitInfo 获取指定平台的等待状态信息 func (prl *PlatformRateLimiter) GetPlatformWaitInfo(platform string) WaitInfo { + if !prl.enabled { + return WaitInfo{} + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -254,6 +288,10 @@ func (prl *PlatformRateLimiter) GetPlatformWaitInfo(platform string) WaitInfo { // ForceAccess 强制访问平台，忽略频率限制 // 返回距离上次访问的时间间隔 func (prl *PlatformRateLimiter) ForceAccess(platform string) time.Duration { + if !prl.enabled { + return 0 + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() diff --git a/src/pkg/ratelimit/ratelimit_test.go b/src/pkg/ratelimit/ratelimit_test.go index 6df3932..f7bee3a 100644 --- a/src/pkg/ratelimit/ratelimit_test.go +++ b/src/pkg/ratelimit/ratelimit_test.go @@ -7,7 +7,7 @@ import ( ) func TestPlatformRateLimiter(t *testing.T) { - limiter := GetGlobalRateLimiter() + limiter := newPlatformRateLimiter(true) // 设置测试平台限制：2秒间隔 limiter.SetPlatformLimit("test_platform", 2) @@ -44,7 +44,7 @@ func TestPlatformRateLimiter(t *testing.T) { } func TestPlatformRateLimiterUpdate(t *testing.T) { - limiter := GetGlobalRateLimiter() + limiter := newPlatformRateLimiter(true) // 设置初始限制 limiter.SetPlatformLimit("update_test", 3) @@ -80,6 +80,7 @@ func TestAcquirePlatformSerializesInFlightRequests(t *testing.T) { inFlight: make(chan struct{}, 1), }, }, + enabled: true, } releaseFirst, ok := limiter.AcquirePlatformWithContext(context.Background(), "test") @@ -112,7 +113,7 @@ func TestAcquirePlatformSerializesInFlightRequests(t *testing.T) { } func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) { - limiter := &PlatformRateLimiter{limiters: make(map[string]*PlatformLimiter)} + limiter := newPlatformRateLimiter(true) limiter.SetPlatformLimit("test", 5) limiter.EnsurePlatformLimit("test", 1) @@ -120,3 +121,23 @@ func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) { t.Fatalf("兜底限制覆盖了显式配置：得到 %d 秒，期望 5 秒", got) } } + +func TestGlobalPlatformRateLimiterDisabled(t *testing.T) { + limiter := GetGlobalRateLimiter() + limiter.SetPlatformLimit("test", 60) + + if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { + t.Fatalf("全局平台限制器应保持关闭，实际限制：%v", limits) + } + + release, ok := limiter.AcquirePlatformWithContext(context.Background(), "test") + if !ok { + t.Fatal("关闭平台限制后请求应立即获准") + } + release() + + waitInfo := limiter.GetPlatformWaitInfo("test") + if waitInfo.MinIntervalSec != 0 || waitInfo.NextRequestInSec != 0 { + t.Fatalf("关闭平台限制后不应报告等待状态：%+v", waitInfo) + } +} From 45dcc52ba3be3da6d074b41fa26187458b2528f7 Mon Sep 17 00:00:00 2001 From: kira1928 Date: Sun, 23 Aug 2026 04:09:33 +0900 Subject: [PATCH 51/52] =?UTF-8?q?fix:=20=E5=90=8C=E6=AD=A5=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E9=99=90=E6=B5=81=E7=A6=81=E7=94=A8=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pkg/ratelimit/ratelimit.go | 13 +++ src/pkg/ratelimit/ratelimit_test.go | 12 +++ src/servers/handler.go | 28 ++++-- src/servers/handler_test.go | 38 ++++++++ .../src/component/config-info/index.tsx | 40 ++++++-- src/webapp/src/component/live-list/index.tsx | 93 +++++++++++++++---- 6 files changed, 193 insertions(+), 31 deletions(-) diff --git a/src/pkg/ratelimit/ratelimit.go b/src/pkg/ratelimit/ratelimit.go index cd895cf..beadcd9 100644 --- a/src/pkg/ratelimit/ratelimit.go +++ b/src/pkg/ratelimit/ratelimit.go @@ -40,6 +40,11 @@ func GetGlobalRateLimiter() *PlatformRateLimiter { return globalRateLimiter } +// Enabled 返回当前限制器是否实际应用平台级访问限制。 +func (prl *PlatformRateLimiter) Enabled() bool { + return prl.enabled +} + // SetPlatformLimit 设置或更新指定平台的访问频率限制 func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec int) { if !prl.enabled { @@ -104,6 +109,10 @@ func (prl *PlatformRateLimiter) WaitForPlatform(platform string) { // 如果平台没有设置限制，立即返回 true // 返回 true 表示成功获取访问权限，false 表示被 context 取消 func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, platform string) bool { + if ctx.Err() != nil { + return false + } + if !prl.enabled { return true } @@ -126,6 +135,10 @@ func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, // 仅限制请求开始间隔不足以保护启动阶段：当前一个请求耗时超过最小间隔时，后续请求仍会 // 重叠执行。这里把并发槽位与开始间隔合并为一次许可，使大量直播间并发初始化时仍按平台串行。 func (prl *PlatformRateLimiter) AcquirePlatformWithContext(ctx context.Context, platform string) (release func(), ok bool) { + if ctx.Err() != nil { + return nil, false + } + if !prl.enabled { return func() {}, true } diff --git a/src/pkg/ratelimit/ratelimit_test.go b/src/pkg/ratelimit/ratelimit_test.go index f7bee3a..73c4a13 100644 --- a/src/pkg/ratelimit/ratelimit_test.go +++ b/src/pkg/ratelimit/ratelimit_test.go @@ -125,6 +125,9 @@ func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) { func TestGlobalPlatformRateLimiterDisabled(t *testing.T) { limiter := GetGlobalRateLimiter() limiter.SetPlatformLimit("test", 60) + if limiter.Enabled() { + t.Fatal("全局平台限制器应保持关闭") + } if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { t.Fatalf("全局平台限制器应保持关闭，实际限制：%v", limits) @@ -140,4 +143,13 @@ func TestGlobalPlatformRateLimiterDisabled(t *testing.T) { if waitInfo.MinIntervalSec != 0 || waitInfo.NextRequestInSec != 0 { t.Fatalf("关闭平台限制后不应报告等待状态：%+v", waitInfo) } + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + if limiter.WaitForPlatformWithContext(canceledCtx, "test") { + t.Fatal("关闭平台限制后仍应尊重 context 取消") + } + if release, acquired := limiter.AcquirePlatformWithContext(canceledCtx, "test"); acquired || release != nil { + t.Fatal("context 已取消时不应获取平台许可") + } } diff --git a/src/servers/handler.go b/src/servers/handler.go index 00371f7..0ec6e75 100644 --- a/src/servers/handler.go +++ b/src/servers/handler.go @@ -185,7 +185,13 @@ func getLive(writer http.ResponseWriter, r *http.Request) { } // 获取平台等待状态信息 - waitInfo := ratelimit.GetGlobalRateLimiter().GetPlatformWaitInfo(platformKey) + platformRateLimiter := ratelimit.GetGlobalRateLimiter() + platformRateLimitEnabled := platformRateLimiter.Enabled() + waitInfo := platformRateLimiter.GetPlatformWaitInfo(platformKey) + platformRateLimit := 0 + if platformRateLimitEnabled { + platformRateLimit = cfg.GetPlatformMinAccessInterval(platformKey) + } // 获取调度器状态信息 var schedulerStatus *live.SchedulerStatus @@ -250,10 +256,12 @@ func getLive(writer http.ResponseWriter, r *http.Request) { "audio_only": room.AudioOnly, // 平台访问限制 - "platform_rate_limit": cfg.GetPlatformMinAccessInterval(platformKey), + "platform_rate_limit": platformRateLimit, + "platform_rate_limit_enabled": platformRateLimitEnabled, // 平台等待状态 "rate_limit_info": map[string]interface{}{ + "enabled": platformRateLimitEnabled, "waited_seconds": waitInfo.WaitedSeconds, "next_request_in_sec": waitInfo.NextRequestInSec, "min_interval_sec": waitInfo.MinIntervalSec, @@ -469,7 +477,8 @@ func parseLiveAction(writer http.ResponseWriter, r *http.Request) { case "forceRefresh": // 强制刷新：忽略平台访问频率限制，立即获取最新信息 platformKey := configs.GetPlatformKeyFromUrl(live.GetRawUrl()) - ratelimit.GetGlobalRateLimiter().ForceAccess(platformKey) + platformRateLimiter := ratelimit.GetGlobalRateLimiter() + platformRateLimiter.ForceAccess(platformKey) // 手动调用 GetInfo 获取最新信息 info, err := live.GetInfo() @@ -481,8 +490,9 @@ func parseLiveAction(writer http.ResponseWriter, r *http.Request) { } // 广播频率限制更新事件，通知前端更新倒计时 - waitInfo := ratelimit.GetGlobalRateLimiter().GetPlatformWaitInfo(platformKey) + waitInfo := platformRateLimiter.GetPlatformWaitInfo(platformKey) GetSSEHub().BroadcastRateLimitUpdate(live.GetLiveId(), map[string]interface{}{ + "enabled": platformRateLimiter.Enabled(), "waited_seconds": waitInfo.WaitedSeconds, "next_request_in_sec": waitInfo.NextRequestInSec, "min_interval_sec": waitInfo.MinIntervalSec, @@ -1187,6 +1197,7 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) { }) return } + platformRateLimitEnabled := ratelimit.GetGlobalRateLimiter().Enabled() // 统计每个平台的直播间（只统计正在监控的） platformRooms := make(map[string][]map[string]interface{}) @@ -1269,7 +1280,7 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) { // 检查是否低于最小访问间隔 warningMessage := "" - if listeningCount > 0 && platformConfig.MinAccessIntervalSec > 0 && actualAccessInterval < float64(platformConfig.MinAccessIntervalSec) { + if platformRateLimitEnabled && listeningCount > 0 && platformConfig.MinAccessIntervalSec > 0 && actualAccessInterval < float64(platformConfig.MinAccessIntervalSec) { effectiveInterval := float64(platformConfig.MinAccessIntervalSec) * float64(listeningCount) warningMessage = fmt.Sprintf("当前设置下实际每个直播间的检测间隔约为 %.1f 秒（受最小访问间隔限制）", effectiveInterval) } @@ -1349,9 +1360,10 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) { } response := map[string]interface{}{ - "platforms": stats, - "available_platforms": availablePlatforms, - "global_interval": cfg.Interval, + "platforms": stats, + "available_platforms": availablePlatforms, + "global_interval": cfg.Interval, + "platform_rate_limit_enabled": platformRateLimitEnabled, } writeJSON(writer, response) diff --git a/src/servers/handler_test.go b/src/servers/handler_test.go index f5bbfe7..fb9dc88 100644 --- a/src/servers/handler_test.go +++ b/src/servers/handler_test.go @@ -1,6 +1,7 @@ package servers import ( + "context" "encoding/json" "net/http/httptest" "testing" @@ -8,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/instance" ) func TestGetSoopLiveAuthConfigDoesNotExposeSavedPassword(t *testing.T) { @@ -32,3 +34,39 @@ func TestGetSoopLiveAuthConfigDoesNotExposeSavedPassword(t *testing.T) { _, exists := data["password"] assert.False(t, exists) } + +func TestGetPlatformStatsReportsDisabledRateLimit(t *testing.T) { + cfg := configs.NewConfig() + cfg.Interval = 1 + cfg.PlatformConfigs[configs.PlatformKeyDouyin] = configs.PlatformConfig{ + Name: "抖音", + MinAccessIntervalSec: 60, + } + cfg.LiveRooms = []configs.LiveRoom{{ + Url: "https://live.douyin.com/123456", + IsListening: true, + }} + cfg.RefreshLiveRoomIndexCache() + configs.SetCurrentConfig(cfg) + t.Cleanup(func() { configs.SetCurrentConfig(configs.NewConfig()) }) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/api/config/platforms", nil) + request = request.WithContext(context.WithValue(request.Context(), instance.Key, &instance.Instance{})) + getPlatformStats(recorder, request) + + assert.Equal(t, 200, recorder.Code) + var response struct { + PlatformRateLimitEnabled bool `json:"platform_rate_limit_enabled"` + Platforms []struct { + PlatformKey string `json:"platform_key"` + WarningMessage string `json:"warning_message"` + } `json:"platforms"` + } + assert.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &response)) + assert.False(t, response.PlatformRateLimitEnabled) + if assert.Len(t, response.Platforms, 1) { + assert.Equal(t, configs.PlatformKeyDouyin, response.Platforms[0].PlatformKey) + assert.Empty(t, response.Platforms[0].WarningMessage) + } +} diff --git a/src/webapp/src/component/config-info/index.tsx b/src/webapp/src/component/config-info/index.tsx index 0f7bbc6..f51b9b6 100644 --- a/src/webapp/src/component/config-info/index.tsx +++ b/src/webapp/src/component/config-info/index.tsx @@ -183,6 +183,7 @@ interface PlatformStatsResponse { platforms: PlatformStat[]; available_platforms: string[]; global_interval: number; + platform_rate_limit_enabled?: boolean; } // 实际生效值显示组件 @@ -1082,6 +1083,7 @@ const PlatformSettings: React.FC<{ } const { platforms, available_platforms, global_interval } = platformStats; + const platformRateLimitEnabled = platformStats.platform_rate_limit_enabled ?? true; // 分组平台：有直播间的 vs 只有配置没有直播间的 const platformsWithRooms = platforms.filter(p => p.has_rooms); @@ -1186,6 +1188,7 @@ const PlatformSettings: React.FC<{ platform={platform} globalConfig={globalConfig} globalInterval={global_interval} + platformRateLimitEnabled={platformRateLimitEnabled} onSave={(values) => handleSave(platform.platform_key, values)} onDelete={() => handleDelete(platform.platform_key)} loading={loading} @@ -1208,6 +1211,15 @@ const PlatformSettings: React.FC<{ showIcon style={{ marginBottom: 16 }} /> + {!platformRateLimitEnabled && ( + + )} {/* 正在监控的平台 */} {platformsWithRooms.length > 0 && ( @@ -1257,11 +1269,12 @@ const PlatformConfigForm: React.FC<{ platform: PlatformStat; globalConfig: EffectiveConfig; globalInterval: number; + platformRateLimitEnabled: boolean; onSave: (values: any) => void; onDelete: () => void; loading: boolean; onNavigateToRoom: (liveId: string) => void; -}> = ({ platform, globalConfig, globalInterval, onSave, onDelete, loading, onNavigateToRoom }) => { +}> = ({ platform, globalConfig, globalInterval, platformRateLimitEnabled, onSave, onDelete, loading, onNavigateToRoom }) => { const [form] = Form.useForm(); useEffect(() => { @@ -1341,18 +1354,31 @@ const PlatformConfigForm: React.FC<{ 0, inheritedValue: '不限制 (0)', - }} - valueDisplay={(platform.min_access_interval_sec || 0) > 0 ? platform.min_access_interval_sec : '不限制 (0)'} + } : undefined} + effectiveValue={platformRateLimitEnabled ? undefined : '当前不生效'} + valueDisplay={!platformRateLimitEnabled + ? ((platform.min_access_interval_sec || 0) > 0 + ? `已配置 ${platform.min_access_interval_sec} 秒（暂不生效）` + : '暂未启用') + : ((platform.min_access_interval_sec || 0) > 0 ? platform.min_access_interval_sec : '不限制 (0)')} > - + diff --git a/src/webapp/src/component/live-list/index.tsx b/src/webapp/src/component/live-list/index.tsx index 2f77d08..2ddc9b3 100644 --- a/src/webapp/src/component/live-list/index.tsx +++ b/src/webapp/src/component/live-list/index.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { Button, Divider, Table, Tag, Tabs, Row, Col, Tooltip, message, List, Typography, Switch, Space, Popconfirm, Select, Spin } from 'antd'; +import { Alert, Button, Divider, Table, Tag, Tabs, Row, Col, Tooltip, message, List, Typography, Switch, Space, Popconfirm, Select, Spin } from 'antd'; import { EditOutlined, SyncOutlined, CloudSyncOutlined, ReloadOutlined, SwapOutlined, CheckCircleOutlined, ExclamationCircleOutlined, CommentOutlined } from '@ant-design/icons'; import PopDialog from '../pop-dialog/index'; import BatchAddRoomDialog from '../batch-add-room-dialog/index'; @@ -824,11 +824,41 @@ class LiveList extends React.Component { // 旧的频率限制信息处理逻辑（兼容性保留） const rateLimitInfo = updateData; + const rateLimitEnabled = rateLimitInfo?.enabled + ?? currentDetail?.platform_rate_limit_enabled + ?? true; const updatedDetail = { ...currentDetail, - rate_limit_info: rateLimitInfo + platform_rate_limit_enabled: rateLimitEnabled, + platform_rate_limit: rateLimitEnabled ? currentDetail?.platform_rate_limit : 0, + rate_limit_info: { + ...rateLimitInfo, + enabled: rateLimitEnabled + } }; + if (!rateLimitEnabled) { + return { + ...prevState, + expandedDetails: { + ...prevState.expandedDetails, + [roomId]: updatedDetail + }, + countdownTimers: { + ...prevState.countdownTimers, + [roomId]: 0 + }, + lastUpdateTimes: { + ...prevState.lastUpdateTimes, + [roomId]: Date.now() + }, + refreshStatus: { + ...prevState.refreshStatus, + [roomId]: 'idle' + } + }; + } + const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0); const minIntervalSec = rateLimitInfo?.min_interval_sec || currentDetail?.platform_rate_limit || 20; const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0); @@ -1235,6 +1265,9 @@ class LiveList extends React.Component { // 优先使用 scheduler_status 来确定刷新状态 const schedulerStatus = detail.scheduler_status; const rateLimitInfo = detail.rate_limit_info; + const rateLimitEnabled = detail.platform_rate_limit_enabled + ?? rateLimitInfo?.enabled + ?? true; let initialCountdown = 0; let initialStatus: RefreshStatus = 'idle'; @@ -1249,14 +1282,14 @@ class LiveList extends React.Component { // 有下次请求计划 initialCountdown = Math.ceil(schedulerStatus.seconds_until_next_request); // 检查是否在等待平台限制 - if (rateLimitInfo?.next_request_in_sec > 0) { + if (rateLimitEnabled && rateLimitInfo?.next_request_in_sec > 0) { initialStatus = 'waiting_rate_limit'; } else { initialStatus = 'waiting_interval'; } } else if (schedulerStatus.seconds_until_next_request === 0) { // 即将发送请求或正在等待平台限制 - if (rateLimitInfo?.next_request_in_sec > 0) { + if (rateLimitEnabled && rateLimitInfo?.next_request_in_sec > 0) { initialCountdown = Math.ceil(rateLimitInfo.next_request_in_sec); initialStatus = 'waiting_rate_limit'; } else { @@ -1270,19 +1303,24 @@ class LiveList extends React.Component { } } else { // 回退到旧逻辑（兼容性） - const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0); - const minIntervalSec = rateLimitInfo?.min_interval_sec || detail.platform_rate_limit || 20; - const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0); - - if (nextRequestInSec > 0) { - initialCountdown = nextRequestInSec; - initialStatus = 'waiting_rate_limit'; - } else if (waitedSec < minIntervalSec) { - initialCountdown = minIntervalSec - waitedSec; - initialStatus = 'waiting_interval'; - } else { + if (!rateLimitEnabled) { initialCountdown = 0; initialStatus = 'idle'; + } else { + const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0); + const minIntervalSec = rateLimitInfo?.min_interval_sec || detail.platform_rate_limit || 20; + const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0); + + if (nextRequestInSec > 0) { + initialCountdown = nextRequestInSec; + initialStatus = 'waiting_rate_limit'; + } else if (waitedSec < minIntervalSec) { + initialCountdown = minIntervalSec - waitedSec; + initialStatus = 'waiting_interval'; + } else { + initialCountdown = 0; + initialStatus = 'idle'; + } } } @@ -1428,6 +1466,9 @@ class LiveList extends React.Component { const countdown = countdownTimers[record.roomId] ?? 0; const status = refreshStatus[record.roomId] ?? 'idle'; const liveId = record.roomId; + const platformRateLimitEnabled = detail?.platform_rate_limit_enabled + ?? detail?.rate_limit_info?.enabled + ?? true; // 保存 this 引用供嵌套函数使用 const component = this; @@ -1782,7 +1823,27 @@ class LiveList extends React.Component { 平台访问频率控制
- {detail.rate_limit_info ? ( + {!platformRateLimitEnabled ? ( +
+ +
+ +
+
+ ) : detail.rate_limit_info ? (
Date: Fri, 4 Sep 2026 00:49:03 +0900 Subject: [PATCH 52/52] =?UTF-8?q?fix:=20=E6=B6=88=E9=99=A4=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=BF=AB=E7=85=A7=E5=B9=B6=E5=8F=91=E8=AF=BB=E5=86=99?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E7=9A=84=20fatal=20=E5=B4=A9=E6=BA=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetLiveRoomByUrl 在索引缓存未命中时会调用 RefreshLiveRoomIndexCache 写入共享的不可变配置快照的 map。该函数被列表接口、录制、监听等大量 goroutine 高频并发调用，一个 goroutine 写 map 而另一个读同一 map 时， 触发 Go 运行时的 fatal error（concurrent map read and map write）， 整个进程直接退出——表现为「开始录制后一段时间报错崩溃」。 配置采用不可变快照架构（Update 复制-修改-原子替换，替换前统一 RefreshLiveRoomIndexCache），已存入的快照不应再被就地修改。本次修复： - GetLiveRoomByUrl 改为纯只读：命中缓存直接返回，未命中回退到只读线性 扫描，不再写共享快照的 map。 - 移除 putConfig / putRawConfig 中对共享快照的 RefreshLiveRoomIndexCache 调用（Marshal 不需要该缓存，后续查找也已是只读）。 - 删除无调用者且同样就地修改快照的死方法 (*Config).RemoveLiveRoomByUrl。 - 新增并发回归测试 TestGetLiveRoomByUrlConcurrent。 via [HAPI](https://hapi.run) Co-Authored-By: HAPI Co-Authored-By: Claude Opus 4.8 --- src/configs/config.go | 30 ++++++++++++------------------ src/configs/config_test.go | 35 +++++++++++++++++++++++++++++++++++ src/servers/handler.go | 6 ++++-- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/configs/config.go b/src/configs/config.go index d76302a..8a06247 100644 --- a/src/configs/config.go +++ b/src/configs/config.go @@ -1170,27 +1170,21 @@ func (c *Config) RefreshLiveRoomIndexCache() { } } -func (c *Config) RemoveLiveRoomByUrl(url string) error { - c.RefreshLiveRoomIndexCache() - if index, ok := c.liveRoomIndexCache[url]; ok { - if index >= 0 && index < len(c.LiveRooms) && c.LiveRooms[index].Url == url { - c.LiveRooms = append(c.LiveRooms[:index], c.LiveRooms[index+1:]...) - delete(c.liveRoomIndexCache, url) - return nil - } - } - return errors.New("failed removing room: " + url) -} - func (c *Config) GetLiveRoomByUrl(url string) (*LiveRoom, error) { - room, err := c.getLiveRoomByUrlImpl(url) - if err != nil { - c.RefreshLiveRoomIndexCache() - if room, err = c.getLiveRoomByUrlImpl(url); err != nil { - return nil, err + // 配置快照是不可变的：所有写操作都走 Update 的“复制-修改-原子替换”路径， + // 并在替换前调用 RefreshLiveRoomIndexCache 维护索引缓存。因此这里只做只读查找， + // 绝不在共享快照上写 map/切片，否则会与其它 goroutine 的读发生并发读写， + // 触发 Go 运行时的 fatal error（concurrent map read and map write）。 + if room, err := c.getLiveRoomByUrlImpl(url); err == nil { + return room, nil + } + // 索引缓存未命中时回退到只读线性扫描，作为兜底（例如未经 Refresh 的手工构造配置）。 + for i := range c.LiveRooms { + if c.LiveRooms[i].Url == url { + return &c.LiveRooms[i], nil } } - return room, nil + return nil, errors.New("room " + url + " doesn't exist.") } func (c Config) getLiveRoomByUrlImpl(url string) (*LiveRoom, error) { diff --git a/src/configs/config_test.go b/src/configs/config_test.go index 555832d..a44fd24 100644 --- a/src/configs/config_test.go +++ b/src/configs/config_test.go @@ -2,6 +2,7 @@ package configs import ( "os" + "sync" "testing" "github.com/bililive-go/bililive-go/src/pkg/ratelimit" @@ -121,6 +122,40 @@ func TestPlatformRateLimitRemainsDisabledAfterConfigUpdate(t *testing.T) { } } +// TestGetLiveRoomByUrlConcurrent 保证在共享的不可变配置快照上并发查询房间时不会 +// 触发 Go 运行时的 "concurrent map read and map write" fatal error。 +// 回归此前的 bug：GetLiveRoomByUrl 在缓存未命中时会调用 RefreshLiveRoomIndexCache +// 写入共享快照的 map，与其它 goroutine 的读并发时导致崩溃。 +// 使用 -race 运行可进一步检测数据竞争。 +func TestGetLiveRoomByUrlConcurrent(t *testing.T) { + cfg := NewConfig() + cfg.LiveRooms = []LiveRoom{ + {Url: "https://live.bilibili.com/1", IsListening: true}, + {Url: "https://live.bilibili.com/2", IsListening: true}, + } + cfg.RefreshLiveRoomIndexCache() + + var wg sync.WaitGroup + for i := 0; i < 64; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 500; j++ { + // 交替命中缓存与未命中（触发原先的写-回退路径）。 + if (i+j)%2 == 0 { + room, err := cfg.GetLiveRoomByUrl("https://live.bilibili.com/1") + assert.NoError(t, err) + assert.Equal(t, "https://live.bilibili.com/1", room.Url) + } else { + _, err := cfg.GetLiveRoomByUrl("https://live.bilibili.com/does-not-exist") + assert.Error(t, err) + } + } + }(i) + } + wg.Wait() +} + func TestBackwardsCompatibility(t *testing.T) { // Test that old config files still work oldConfigYaml := ` diff --git a/src/servers/handler.go b/src/servers/handler.go index 0ec6e75..78ac66b 100644 --- a/src/servers/handler.go +++ b/src/servers/handler.go @@ -952,8 +952,9 @@ func getConfig(writer http.ResponseWriter, r *http.Request) { } func putConfig(writer http.ResponseWriter, r *http.Request) { + // 直接序列化当前快照即可；索引缓存不参与序列化，且共享快照不可变， + // 不能在此调用 RefreshLiveRoomIndexCache 写它的 map。 config := configs.GetCurrentConfig() - config.RefreshLiveRoomIndexCache() if err := config.Marshal(); err != nil { writeJsonWithStatusCode(writer, http.StatusBadRequest, commonResp{ ErrNo: http.StatusBadRequest, @@ -1002,7 +1003,8 @@ func putRawConfig(writer http.ResponseWriter, r *http.Request) { return } oldConfig := configs.GetCurrentConfig() - oldConfig.RefreshLiveRoomIndexCache() + // 共享快照不可变：其索引缓存在生成时已由 Update 维护好， + // 这里只做只读查找，不能再调用 RefreshLiveRoomIndexCache 写它的 map。 // 继承原配置的文件路径 newConfig.File = oldConfig.File // 预先将旧配置中的 LiveId 迁移到新配置（相同 URL）