diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 97c40a5..d9702c4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -130,6 +130,7 @@ jobs: dist/**/*.zip dist/**/*.7z prerelease: ${{ contains(env.GIT_TAG, 'rc') }} + generate_release_notes: true - name: "[Dry-Run] 跳过 GitHub Release 发布" if: ${{ env.IS_DRY_RUN == 'true' }} run: echo "::notice::Dry-run 模式：跳过 GitHub Release 发布（GIT_TAG=$GIT_TAG）" diff --git a/config.yml b/config.yml index 0cd83aa..6eaad1e 100644 --- a/config.yml +++ b/config.yml @@ -31,22 +31,39 @@ video_split_strategies: on_record_finished: convert_to_mp4: false delete_flv_after_convert: false - # 当 custom_commandline 的值 不为空时，convert_to_mp4 的值会被无视， - # 而是在录制结束后直接执行 custom_commandline 中的命令。 - # 在 custom_commandline 执行结束后，程序还会继续查看 delete_flv_after_convert 的值， - # 来判断是否需要删除原始 flv 文件。 - # 以下是一个在录制结束后将 flv 视频转换为同名 mp4 视频的示例： - # custom_commandline: '{{ .Ffmpeg }} -hide_banner -i "{{ .FileName }}" -c copy "{{ .FileName | trimSuffix (.FileName | ext)}}.mp4"' custom_commandline: "" fix_flv_at_first: true save_cover: false cloud_upload: enable: false storage_name: "" - upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }} + upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} delete_after_upload: false + delete_all_after_upload: false upload_timing: after_process + burn_subtitles: false + burn_subtitles_codec: libx264 + burn_subtitles_crf: "18" + burn_subtitles_preset: medium + burn_delete_ass: false + burn_delete_source: false timeout_in_us: 60000000 +danmaku_enable: false +danmaku: + font_size: 36 + font_name: Microsoft YaHei + scroll_area: full + scroll_time: 10 + resolution: 1920x1080 + outline: 1 + opacity: 128 + record_gift: true + record_douyu_gift: true + record_douyin_gift: true + record_guard: true + record_super_chat: true + guard_position: bottom-left + sc_position: bottom-left live_rooms: # quality参数目前仅B站启用，默认为0 # (B站)0代表原画PRO(HEVC)优先, 其他数值为原画(AVC) @@ -106,6 +123,10 @@ notify: icon: "" # 通知级别（可选）: active/timeSensitive/passive/critical level: "" + wxpusher: + enable: false + appToken: "" + uids: [] app_data_path: .appdata read_only_tool_folder: "" tool_root_folder: "" @@ -126,6 +147,8 @@ proxy: openlist: port: 5244 data_path: "" + username: "" + password: "" update: auto_check: true check_interval_hours: 6 diff --git a/src/cmd/bililive/bililive.go b/src/cmd/bililive/bililive.go index 23d83b8..e511e57 100644 --- a/src/cmd/bililive/bililive.go +++ b/src/cmd/bililive/bililive.go @@ -339,6 +339,8 @@ func main() { // 如果启用了云上传功能，初始化 OpenList 管理器 var openlistManager *openlist.Manager + // 串行化 OpenList 生命周期操作，防止快速 enable/disable 导致服务泄漏 + var openlistLifecycleMu sync.Mutex if config.OnRecordFinished.CloudUpload.Enable { // 获取 OpenList 数据目录 openlistDataPath := config.OpenList.DataPath @@ -357,6 +359,9 @@ func main() { bilisentryPkg.Go(func() { if err := openlistManager.Start(rootCtx); err != nil { logger.WithError(err).Error("启动 OpenList 失败，云上传功能将不可用") + // 启动失败时清除全局指针（CAS：仅当仍为当前实例时才清除，避免误清新 Manager） + openlist.ClearGlobalManagerIf(openlistManager) + openlistManager = nil } }) @@ -366,6 +371,113 @@ func main() { logger.Info("云上传功能已启用") } + // 注册配置更新回调：当用户通过 Web UI 首次启用云上传时，自动创建并启动 OpenList Manager + configs.RegisterAfterUpdate(func(oldCfg, newCfg *configs.Config) { + if newCfg == nil { + return + } + wasEnabled := oldCfg != nil && oldCfg.OnRecordFinished.CloudUpload.Enable + isEnabled := newCfg.OnRecordFinished.CloudUpload.Enable + + // 关闭云上传：停止 OpenList 进程 + if wasEnabled && !isEnabled { + openlistLifecycleMu.Lock() + defer openlistLifecycleMu.Unlock() + if mgr := openlist.GetGlobalManager(); mgr != nil { + logger.Info("检测到云上传已关闭，正在停止 OpenList...") + mgr.Stop() + servers.SetOpenListManager(nil) + openlistManager = nil + } + return + } + + // 开启云上传：启动 OpenList 进程 + if wasEnabled || !isEnabled { + // 云上传保持启用：检查是否需要重建 Manager（端口或数据目录变更） + if wasEnabled && isEnabled { + if mgr := openlist.GetGlobalManager(); mgr != nil { + // 解析新的配置值 + newPort := newCfg.OpenList.Port + if newPort == 0 { + newPort = 5244 + } + newDataPath := newCfg.OpenList.DataPath + if newDataPath == "" { + newDataPath = filepath.Join(newCfg.AppDataPath, "openlist") + } + // 与 Manager 当前值比较 + if mgr.GetPort() != newPort || mgr.GetDataPath() != newDataPath { + logger.Infof("检测到 OpenList 配置变更（端口: %d→%d, 数据目录: %s→%s），正在重建 Manager...", + mgr.GetPort(), newPort, mgr.GetDataPath(), newDataPath) + openlistLifecycleMu.Lock() + mgr.Stop() + servers.SetOpenListManager(nil) + // 创建新 Manager 并启动 + newMgr := openlist.NewManager(newDataPath, newPort) + servers.SetOpenListManager(newMgr) + openlistManager = newMgr + openlistLifecycleMu.Unlock() + bilisentryPkg.Go(func() { + if err := newMgr.Start(rootCtx); err != nil { + logger.WithError(err).Error("重建 OpenList 失败") + openlistLifecycleMu.Lock() + // CAS：仅当全局仍为 newMgr 时才清除 + openlist.ClearGlobalManagerIf(newMgr) + openlistManager = nil + openlistLifecycleMu.Unlock() + } + }) + } + } + } + return + } + + openlistLifecycleMu.Lock() + // 已有 Manager 则跳过 + if openlist.GetGlobalManager() != nil { + openlistLifecycleMu.Unlock() + return + } + + logger.Info("检测到云上传已启用，正在初始化 OpenList...") + + openlistDataPath := newCfg.OpenList.DataPath + if openlistDataPath == "" { + openlistDataPath = filepath.Join(newCfg.AppDataPath, "openlist") + } + openlistPort := newCfg.OpenList.Port + if openlistPort == 0 { + openlistPort = 5244 + } + + mgr := openlist.NewManager(openlistDataPath, openlistPort) + // 先设置全局 Manager，让 API 端能立即获取到状态 + servers.SetOpenListManager(mgr) + openlistLifecycleMu.Unlock() + + // 在后台启动 OpenList 进程（需要 rootCtx 生命周期） + bilisentryPkg.Go(func() { + if err := mgr.Start(rootCtx); err != nil { + logger.WithError(err).Error("动态启动 OpenList 失败，云上传功能将不可用") + // CAS：仅当全局仍为当前实例时才清除，避免误清新 Manager + openlistLifecycleMu.Lock() + openlist.ClearGlobalManagerIf(mgr) + openlistLifecycleMu.Unlock() + return + } + // Start 完成后复核配置：如果在 waitForReady 期间配置已关闭，立即清理 + openlistLifecycleMu.Lock() + defer openlistLifecycleMu.Unlock() + if cfg := configs.GetCurrentConfig(); cfg == nil || !cfg.OnRecordFinished.CloudUpload.Enable { + logger.Info("OpenList 启动完成但云上传已关闭，立即停止") + mgr.Stop() + openlist.ClearGlobalManagerIf(mgr) + } + }) + }) + // 初始化 Pipeline 管道管理器 pipelineDbPath := filepath.Join(config.AppDataPath, "db", "pipeline.db") pipelineStore, err := pipeline.NewSQLiteStore(pipelineDbPath) @@ -714,6 +826,11 @@ func main() { if inst.PipelineManager != nil { inst.PipelineManager.Close(ctx) } + // 等待异步通知 goroutine（如 Telegram/Email 摘要发送）完成 + // 必须在 PipelineManager.Close 之后——pipeline 回调可能触发最后的通知 + if rm, ok := inst.RecorderManager.(recorders.Manager); ok { + rm.WaitForNotifications() + } // 关闭直播间状态管理器 if liveStateManager != nil { liveStateManager.Close() diff --git a/src/configs/config.go b/src/configs/config.go index 026bc58..8a06247 100644 --- a/src/configs/config.go +++ b/src/configs/config.go @@ -354,6 +354,7 @@ type CloudUpload struct { UploadPathTmpl string `yaml:"upload_path_tmpl" json:"upload_path_tmpl"` // 上传路径模板 DeleteAfterUpload bool `yaml:"delete_after_upload" json:"delete_after_upload"` // 上传成功后仅删除已上传的文件 DeleteAllAfterUpload bool `yaml:"delete_all_after_upload" json:"delete_all_after_upload"` // 上传成功后删除全部文件（含中间产物） + UploadSubtitles bool `yaml:"upload_subtitles" json:"upload_subtitles"` // 是否上传关联的 .ass 弹幕字幕文件 AdditionalStorages []string `yaml:"additional_storages,omitempty" json:"additional_storages,omitempty"` // 额外存储（支持多目标上传） } @@ -618,6 +619,36 @@ var updateMu sync.Mutex // 当期望版本与实际版本不一致时返回的错误 var ErrConfigVersionConflict = errors.New("config version conflict") +// AfterUpdateCallback 配置更新后的回调函数类型 +// old 为更新前的配置（可能为 nil），newCfg 为更新后的配置 +type AfterUpdateCallback func(old, newCfg *Config) + +// afterUpdateCallbacks 配置更新后的回调列表 +var ( + afterUpdateCallbacks []AfterUpdateCallback + afterUpdateMu sync.RWMutex +) + +// RegisterAfterUpdate 注册配置更新后的回调 +// 回调在配置更新并持久化完成后同步执行，此时已释放 updateMu 锁 +func RegisterAfterUpdate(cb AfterUpdateCallback) { + afterUpdateMu.Lock() + defer afterUpdateMu.Unlock() + afterUpdateCallbacks = append(afterUpdateCallbacks, cb) +} + +// fireAfterUpdate 触发所有配置更新回调 +func fireAfterUpdate(old, newCfg *Config) { + afterUpdateMu.RLock() + callbacks := make([]AfterUpdateCallback, len(afterUpdateCallbacks)) + copy(callbacks, afterUpdateCallbacks) + afterUpdateMu.RUnlock() + + for _, cb := range callbacks { + cb(old, newCfg) + } +} + func SetCurrentConfig(cfg *Config) { if cfg == nil { // 存储 nil 以保持行为一致 @@ -657,19 +688,19 @@ func UpdateTransient(mutator func(c *Config) error) (*Config, error) { } func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) { - var newCfg *Config + var oldCfg, newCfg *Config var updateErr error func() { updateMu.Lock() defer updateMu.Unlock() - old := GetCurrentConfig() + oldCfg = GetCurrentConfig() // 若当前尚未设置配置，则以默认配置为基础 var base *Config - if old == nil { + if oldCfg == nil { base = NewConfig() } else { - base = CloneConfigShallow(old) + base = CloneConfigShallow(oldCfg) } if err := mutator(base); err != nil { updateErr = err @@ -678,10 +709,10 @@ func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) { // 维护派生字段 base.RefreshLiveRoomIndexCache() // 版本号自增 - if old == nil { + if oldCfg == nil { base.Version = 1 } else { - base.Version = old.Version + 1 + base.Version = oldCfg.Version + 1 } newCfg = base @@ -702,6 +733,7 @@ func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) { return nil, errors.New("config update failed") } + fireAfterUpdate(oldCfg, newCfg) return newCfg, nil } @@ -712,17 +744,17 @@ func UpdateCAS(expectedVersion int64, mutator func(c *Config) error) (*Config, e } func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist bool) (*Config, error) { - var newCfg *Config + var oldCfg, newCfg *Config var updateErr error func() { updateMu.Lock() defer updateMu.Unlock() - cur := GetCurrentConfig() + oldCfg = GetCurrentConfig() // 校验版本 var curVersion int64 - if cur != nil { - curVersion = cur.Version + if oldCfg != nil { + curVersion = oldCfg.Version } if curVersion != expectedVersion { updateErr = ErrConfigVersionConflict @@ -730,10 +762,10 @@ func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist } // 克隆并修改 var base *Config - if cur == nil { + if oldCfg == nil { base = NewConfig() } else { - base = CloneConfigShallow(cur) + base = CloneConfigShallow(oldCfg) } if err := mutator(base); err != nil { updateErr = err @@ -757,6 +789,7 @@ func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist return nil, updateErr } + fireAfterUpdate(oldCfg, newCfg) return newCfg, nil } @@ -980,7 +1013,7 @@ var defaultConfig = Config{ CloudUpload: CloudUpload{ Enable: false, StorageName: "", - UploadPathTmpl: "/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date \"2006-01-02\" }}.{{ .Ext }}", + UploadPathTmpl: "/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }}", DeleteAfterUpload: false, }, UploadTiming: UploadTimingAfterProcess, @@ -1137,27 +1170,21 @@ func (c *Config) RefreshLiveRoomIndexCache() { } } -func (c *Config) RemoveLiveRoomByUrl(url string) error { - c.RefreshLiveRoomIndexCache() - if index, ok := c.liveRoomIndexCache[url]; ok { - if index >= 0 && index < len(c.LiveRooms) && c.LiveRooms[index].Url == url { - c.LiveRooms = append(c.LiveRooms[:index], c.LiveRooms[index+1:]...) - delete(c.liveRoomIndexCache, url) - return nil - } - } - return errors.New("failed removing room: " + url) -} - func (c *Config) GetLiveRoomByUrl(url string) (*LiveRoom, error) { - room, err := c.getLiveRoomByUrlImpl(url) - if err != nil { - c.RefreshLiveRoomIndexCache() - if room, err = c.getLiveRoomByUrlImpl(url); err != nil { - return nil, err + // 配置快照是不可变的：所有写操作都走 Update 的“复制-修改-原子替换”路径， + // 并在替换前调用 RefreshLiveRoomIndexCache 维护索引缓存。因此这里只做只读查找， + // 绝不在共享快照上写 map/切片，否则会与其它 goroutine 的读发生并发读写， + // 触发 Go 运行时的 fatal error（concurrent map read and map write）。 + if room, err := c.getLiveRoomByUrlImpl(url); err == nil { + return room, nil + } + // 索引缓存未命中时回退到只读线性扫描，作为兜底（例如未经 Refresh 的手工构造配置）。 + for i := range c.LiveRooms { + if c.LiveRooms[i].Url == url { + return &c.LiveRooms[i], nil } } - return room, nil + return nil, errors.New("room " + url + " doesn't exist.") } func (c Config) getLiveRoomByUrlImpl(url string) (*LiveRoom, error) { diff --git a/src/configs/config_comments.go b/src/configs/config_comments.go index b8c0a24..830eac9 100644 --- a/src/configs/config_comments.go +++ b/src/configs/config_comments.go @@ -32,61 +32,6 @@ func DecorateConfigNode(node *yaml.Node) { # 负数为非法值，程序会输出 log 提醒，并无视所设定的数值`, "") } - finishNode := findNode(root, "on_record_finished") - if finishNode != nil { - setFieldComment(finishNode, "custom_commandline", - `# 当 custom_commandline 的值 不为空时，convert_to_mp4 的值会被无视， -# 而是在录制结束后直接执行 custom_commandline 中的命令。 -# 在 custom_commandline 执行结束后，程序还会继续查看 delete_flv_after_convert 的值， -# 来判断是否需要删除原始 flv 文件。 -# 以下是一个在录制结束后将 flv 视频转换为同名 mp4 视频的示例： -# custom_commandline: '{{ .Ffmpeg }} -hide_banner -i "{{ .FileName }}" -c copy "{{ .FileName | trimSuffix (.FileName | ext)}}.mp4"'`, "") - - setFieldComment(finishNode, "delete_flv_after_convert", - `# MP4 转换成功后标记原始 FLV 为待删除，全部处理阶段完成后才真正删除 -# 默认 false（保留原始 FLV 文件）`, "") - - setFieldComment(finishNode, "burn_subtitles", - `# 是否将 ASS 弹幕字幕硬编码（烧录）到视频中 -# 开启后会使用 FFmpeg 重编码视频，将字幕叠加到画面上 -# 需要同时开启弹幕录制（danmaku_enable）才有 ASS 文件可烧录`, "") - - setFieldComment(finishNode, "burn_subtitles_codec", - `# 烧录字幕时使用的视频编码器 -# 默认 libx264（H.264），兼容性最好 -# 可选 libx265（H.265，压缩率更高但编码更慢）`, "") - - setFieldComment(finishNode, "burn_subtitles_crf", - `# 烧录字幕时的 CRF（恒定速率因子）质量值 -# 范围 0-51，数值越小画质越好，文件越大 -# 默认 18（视觉无损），推荐范围 15-23`, "") - - setFieldComment(finishNode, "burn_subtitles_preset", - `# 烧录字幕时的编码预设 -# 影响编码速度和压缩率的平衡 -# 可选: ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow -# 默认 medium，越慢画质越好但耗时越长`, "") - - setFieldComment(finishNode, "burn_delete_ass", - `# 烧录成功后标记 ASS 文件为待删除，全部处理阶段完成后才真正删除 -# 默认 false（保留 ASS 文件）`, "") - - setFieldComment(finishNode, "burn_delete_source", - `# 烧录成功后标记源视频文件为待删除，全部处理阶段完成后才真正删除 -# 默认 false（保留源文件，同时存在源文件和烧录后的 MKV） -# 开启后仅保留烧录完成的 MKV 文件`, "") - - cloudNode := findNode(finishNode, "cloud_upload") - if cloudNode != nil { - setFieldComment(cloudNode, "delete_after_upload", - `# 选「处理完再上传」时，上传成功后仅删除已上传的文件（如最终视频），不影响其他中间文件 -# 选「先上传再处理」时此开关无效`, "") - setFieldComment(cloudNode, "delete_all_after_upload", - `# 选「处理完再上传」时，上传成功后删除所有本地文件（含中间产物） -# 选「先上传再处理」时此开关无效`, "") - } - } - setFieldHeadComment(root, "notify", "# 通知服务配置") notifyNode := findNode(root, "notify") if notifyNode != nil { diff --git a/src/configs/config_test.go b/src/configs/config_test.go index eb857ec..a44fd24 100644 --- a/src/configs/config_test.go +++ b/src/configs/config_test.go @@ -2,6 +2,7 @@ package configs import ( "os" + "sync" "testing" "github.com/bililive-go/bililive-go/src/pkg/ratelimit" @@ -97,7 +98,7 @@ func TestGetPlatformMinAccessInterval(t *testing.T) { assert.Equal(t, 1, interval) // 默认最小间隔为 1 秒，防止无限制高频访问 } -func TestDefaultPlatformRateLimitSurvivesTransientConfigUpdate(t *testing.T) { +func TestPlatformRateLimitRemainsDisabledAfterConfigUpdate(t *testing.T) { const roomURL = "https://live.douyin.com/123456" limiter := ratelimit.GetGlobalRateLimiter() t.Cleanup(func() { @@ -109,18 +110,52 @@ func TestDefaultPlatformRateLimitSurvivesTransientConfigUpdate(t *testing.T) { cfg.RefreshLiveRoomIndexCache() SetCurrentConfig(cfg) - if got := limiter.GetAllPlatformLimits()[PlatformKeyDouyin]; got != 1 { - t.Fatalf("初始默认平台限流 = %d 秒，期望 1 秒", got) + if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { + t.Fatalf("配置加载后平台限流应保持关闭，实际限制：%v", limits) } if _, err := SetLiveRoomId(roomURL, types.LiveID("douyin-test")); err != nil { t.Fatalf("写入临时 LiveId 失败: %v", err) } - if got := limiter.GetAllPlatformLimits()[PlatformKeyDouyin]; got != 1 { - t.Fatalf("临时配置更新后的默认平台限流 = %d 秒，期望仍为 1 秒", got) + if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { + t.Fatalf("临时配置更新后平台限流应保持关闭，实际限制：%v", limits) } } +// TestGetLiveRoomByUrlConcurrent 保证在共享的不可变配置快照上并发查询房间时不会 +// 触发 Go 运行时的 "concurrent map read and map write" fatal error。 +// 回归此前的 bug：GetLiveRoomByUrl 在缓存未命中时会调用 RefreshLiveRoomIndexCache +// 写入共享快照的 map，与其它 goroutine 的读并发时导致崩溃。 +// 使用 -race 运行可进一步检测数据竞争。 +func TestGetLiveRoomByUrlConcurrent(t *testing.T) { + cfg := NewConfig() + cfg.LiveRooms = []LiveRoom{ + {Url: "https://live.bilibili.com/1", IsListening: true}, + {Url: "https://live.bilibili.com/2", IsListening: true}, + } + cfg.RefreshLiveRoomIndexCache() + + var wg sync.WaitGroup + for i := 0; i < 64; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 500; j++ { + // 交替命中缓存与未命中（触发原先的写-回退路径）。 + if (i+j)%2 == 0 { + room, err := cfg.GetLiveRoomByUrl("https://live.bilibili.com/1") + assert.NoError(t, err) + assert.Equal(t, "https://live.bilibili.com/1", room.Url) + } else { + _, err := cfg.GetLiveRoomByUrl("https://live.bilibili.com/does-not-exist") + assert.Error(t, err) + } + } + }(i) + } + wg.Wait() +} + func TestBackwardsCompatibility(t *testing.T) { // Test that old config files still work oldConfigYaml := ` diff --git a/src/notify/send.go b/src/notify/send.go index 14db3c4..48bfbc0 100644 --- a/src/notify/send.go +++ b/src/notify/send.go @@ -16,8 +16,9 @@ import ( // RecordingFileDetail 录制文件详情 type RecordingFileDetail struct { - Name string // 文件名（不含路径） - Size int64 // 文件大小（字节） + Name string // 文件名（不含路径） + Size int64 // 文件大小（字节） + Uploaded bool // 是否已上传到云端（用于摘要消息标注） } // SendNotification 发送统一通知函数 @@ -250,6 +251,77 @@ func buildRecordingSummaryMessage(hostName, platform string, files []RecordingFi return } +// buildUploadedSummaryBody 构造"已上传到云端"场景的录制摘要消息体 +// 与 buildRecordingSummaryMessage 类似，但标注文件已上传、不显示磁盘空间 +func buildUploadedSummaryBody(platform string, files []RecordingFileDetail) string { + const maxDisplayFiles = 30 + + var sb strings.Builder + fmt.Fprintf(&sb, "平台：%s\n", platform) + fmt.Fprintf(&sb, "录制文件：%d 个（已上传到云端）\n", len(files)) + var totalSize int64 + for i, f := range files { + totalSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(files) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(files)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "总大小：%s\n", formatFileSize(totalSize)) + fmt.Fprintf(&sb, "本地文件已清理") + return sb.String() +} + +// buildMixedSummaryBody 构造"部分上传、部分本地保留"场景的消息体 +func buildMixedSummaryBody(platform string, uploaded, kept []RecordingFileDetail, outputPath string) string { + const maxDisplayFiles = 30 + + var sb strings.Builder + fmt.Fprintf(&sb, "平台：%s\n", platform) + + // 已上传部分 + fmt.Fprintf(&sb, "已上传到云端：%d 个\n", len(uploaded)) + var uploadedSize int64 + for i, f := range uploaded { + uploadedSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(uploaded) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(uploaded)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "上传总大小：%s\n", formatFileSize(uploadedSize)) + + // 本地保留部分 + fmt.Fprintf(&sb, "本地保留：%d 个\n", len(kept)) + var keptSize int64 + for i, f := range kept { + keptSize += f.Size + if i < maxDisplayFiles { + fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size)) + } + } + if len(kept) > maxDisplayFiles { + fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(kept)-maxDisplayFiles) + } + fmt.Fprintf(&sb, "本地总大小：%s", formatFileSize(keptSize)) + if outputPath != "" { + if free, err := getDiskFreeSpace(outputPath); err == nil { + fmt.Fprintf(&sb, "\n剩余磁盘空间：%s", formatFileSize(int64(free))) + } + } + return sb.String() +} + +// buildRecordingSummaryMessageBody 构造纯本地保留的消息体（不含 title） +func buildRecordingSummaryMessageBody(platform string, files []RecordingFileDetail, outputPath string) string { + _, body := buildRecordingSummaryMessage("", platform, files, outputPath) + return body +} + // SendRecordingSummary 录制结束后发送录制文件摘要通知 // outputPath 为录制输出路径，用于获取剩余磁盘空间 func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform string, files []RecordingFileDetail, outputPath string) { @@ -262,7 +334,59 @@ func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform stri } title, body := buildRecordingSummaryMessage(hostName, platform, files, outputPath) + sendToAllChannels(cfg, logger, title, body) +} +// SendPipelineRecordingSummary Pipeline 完成后发送录制摘要通知 +// originalFiles: 原始录制文件列表（allUploaded 时用于显示） +// finalFiles: Pipeline 处理后的最终文件列表（含 Uploaded 标记） +func SendPipelineRecordingSummary( + logger *livelogger.LiveLogger, + hostName, platform string, + originalFiles, finalFiles []RecordingFileDetail, + outputPath string, +) { + cfg := configs.GetCurrentConfig() + if cfg == nil || !cfg.Notify.SendRecordingSummary { + return + } + + // 分离已上传文件和本地保留文件 + var uploaded []RecordingFileDetail + var kept []RecordingFileDetail + for _, f := range finalFiles { + if f.Uploaded { + uploaded = append(uploaded, f) + } else { + kept = append(kept, f) + } + } + + var title, body string + if len(uploaded) > 0 && len(kept) == 0 { + // 所有文件已上传：显示上传文件列表 + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildUploadedSummaryBody(platform, uploaded) + } else if len(uploaded) > 0 && len(kept) > 0 { + // 部分上传、部分保留：显示两段 + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildMixedSummaryBody(platform, uploaded, kept, outputPath) + } else if len(kept) > 0 { + // 全部本地保留 + title, body = buildRecordingSummaryMessage(hostName, platform, kept, outputPath) + } else if len(originalFiles) > 0 { + // finalFiles 为空，回退到 originalFiles + title = fmt.Sprintf("%s 录制完成", hostName) + body = buildRecordingSummaryMessageBody(platform, originalFiles, outputPath) + } else { + return + } + + sendToAllChannels(cfg, logger, title, body) +} + +// sendToAllChannels 向所有已启用的通知通道推送摘要消息 +func sendToAllChannels(cfg *configs.Config, logger *livelogger.LiveLogger, title, body string) { // Telegram if cfg.Notify.Telegram.Enable { msg := fmt.Sprintf("%s\n%s", title, body) diff --git a/src/pipeline/config.go b/src/pipeline/config.go index 6c33549..77bf36a 100644 --- a/src/pipeline/config.go +++ b/src/pipeline/config.go @@ -32,6 +32,8 @@ const ( OptionCommand = "command" // OptionFileTypes 处理的文件类型过滤 OptionFileTypes = "file_types" + // OptionUploadSubtitles 是否上传关联的 .ass 弹幕字幕文件 + OptionUploadSubtitles = "upload_subtitles" // OptionCodec 视频编码器 OptionCodec = "codec" // OptionCrf CRF 质量值 @@ -85,6 +87,7 @@ func ConvertLegacyConfig(legacy *configs.OnRecordFinished) *PipelineConfig { OptionDeleteAllAfter: legacy.CloudUpload.DeleteAllAfterUpload, OptionUploadTiming: string(legacy.UploadTiming), OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)}, + OptionUploadSubtitles: legacy.CloudUpload.UploadSubtitles, }, } hasCloudUpload := legacy.CloudUpload.Enable && legacy.CloudUpload.StorageName != "" diff --git a/src/pipeline/executor.go b/src/pipeline/executor.go index 7d8e3ab..3c3c716 100644 --- a/src/pipeline/executor.go +++ b/src/pipeline/executor.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/pkg/metadata" bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry" "github.com/sirupsen/logrus" ) @@ -48,11 +50,15 @@ func (e *Executor) getFactory(name string) (StageFactory, bool) { } // Execute 执行管道 +// startStage: 从第几个启用的阶段开始执行（0=全部执行，用于重试指定阶段） +// onStageComplete: 每个阶段成功完成后的回调，用于持久化进度（崩溃恢复的关键） func (e *Executor) Execute( ctx *PipelineContext, config *PipelineConfig, initialFiles []FileInfo, + startStage int, onProgress func(stageIndex int, stageName string, status StageStatus), + onStageComplete func(stageIndex int, result StageResult), ) ([]StageResult, error) { if config == nil || len(config.Stages) == 0 { e.logger.Debug("pipeline config is empty, skipping") @@ -61,6 +67,7 @@ func (e *Executor) Execute( files := initialFiles results := make([]StageResult, 0, len(config.Stages)) + ctx.UploadedPaths = make(map[string]bool) stageIndex := 0 for i, stageCfg := range config.Stages { @@ -76,6 +83,12 @@ func (e *Executor) Execute( continue } + // 跳过 startStage 之前的阶段（用于从指定阶段重试） + if stageIndex < startStage { + stageIndex++ + continue + } + // 记录开始 if onProgress != nil { onProgress(stageIndex, stageCfg.Name, StageStatusRunning) @@ -86,6 +99,9 @@ func (e *Executor) Execute( var commands []string var logs string + // 记录阶段开始时间 + stageStartTime := getTimeNow() + // 并行阶段处理 if stageCfg.IsParallel() { e.logger.WithField("stage_index", i).Debug("executing parallel stages") @@ -99,6 +115,9 @@ func (e *Executor) Execute( output, commands, logs, err = e.executeStage(ctx, stageCfg, files) } + // 回填新产出文件的 Size（阶段内构造 FileInfo 时可能未 stat） + backfillFileSizes(output) + // 记录结果 result := StageResult{ StageName: stageCfg.Name, @@ -107,7 +126,7 @@ func (e *Executor) Execute( Commands: commands, Logs: logs, } - result.StartedAt = getTimeNow() + result.StartedAt = stageStartTime if err != nil { result.Status = StageStatusFailed @@ -134,6 +153,11 @@ func (e *Executor) Execute( onProgress(stageIndex, stageCfg.Name, StageStatusCompleted) } + // 通知外部持久化阶段完成状态（用于崩溃恢复） + if onStageComplete != nil { + onStageComplete(stageIndex, result) + } + // 更新文件列表给下一阶段 files = output stageIndex++ @@ -144,32 +168,143 @@ func (e *Executor) Execute( }).Debug("stage completed") } - // 全部成功：执行延迟删除 - e.deleteMarkedFiles(files) + // 去重：多个阶段可能添加同名文件（如 cloud_upload 上传 .ass 后，burn 又添加 Deletable 副本） + // 保留第一个出现的（保留最早阶段的 Metadata，如 uploaded 标记） + seen := map[string]int{} + var deduped []FileInfo + for _, f := range files { + if idx, exists := seen[f.Path]; exists { + // 合并 Deletable 标记到已有条目 + if f.Deletable && !deduped[idx].Deletable { + deduped[idx].Deletable = true + } + continue + } + seen[f.Path] = len(deduped) + deduped = append(deduped, f) + } + files = deduped + + // 重新标记被后续阶段替换的已上传文件 + // immediate 模式下 cloud_upload 最先执行，后续阶段（fix_flv） + // 创建新 FileInfo 替换原始文件，丢失 Metadata["uploaded"]。 + // 通过 UploadedPaths 按基名+扩展名匹配，为同类型的替换文件恢复上传标记。 + // 不匹配不同扩展名的派生文件（如 .flv 上传后生成的 .mp4/.mkv 不应标记为 uploaded）。 + if len(ctx.UploadedPaths) > 0 { + for uploadedPath := range ctx.UploadedPaths { + uploadedBase := strings.TrimSuffix(filepath.Base(uploadedPath), filepath.Ext(uploadedPath)) + uploadedExt := strings.ToLower(filepath.Ext(uploadedPath)) + for i, f := range files { + if f.Metadata != nil { + if u, ok := f.Metadata["uploaded"].(bool); ok && u { + continue // 已有标记，跳过 + } + } + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != uploadedExt { + continue // 不同扩展名，不是直接替换 + } + fileBase := strings.TrimSuffix(filepath.Base(f.Path), ext) + if fileBase == uploadedBase { + if files[i].Metadata == nil { + files[i].Metadata = map[string]any{} + } + files[i].Metadata["uploaded"] = true + e.logger.Infof("恢复上传标记: %s (匹配 %s)", f.Path, filepath.Base(uploadedPath)) + } + } + } + } + + // 全部成功：执行延迟删除，并让最终结果反映磁盘上的文件 + // 如果上下文已取消，跳过删除以保留所有文件，返回取消错误让任务标记为 Cancelled + if ctx.Ctx.Err() != nil { + e.logger.Warn("pipeline cancelled after completion, skipping file cleanup") + if len(results) > 0 { + results[len(results)-1].OutputFiles = files + } + return results, ctx.Ctx.Err() + } + // 保存最后阶段输出的快照（用于回调提取上传文件详情），再执行清理 + // 必须深拷贝 Metadata，因为 deleteMarkedFiles 会清除 Metadata["uploaded"]， + // FileInfo.Metadata 是 map（引用类型），浅拷贝会共享同一 map + snapshot := make([]FileInfo, len(files)) + for i, f := range files { + cp := f + if f.Metadata != nil { + cp.Metadata = make(map[string]any, len(f.Metadata)) + for k, v := range f.Metadata { + cp.Metadata[k] = v + } + } + snapshot[i] = cp + } + ctx.LastStageFiles = snapshot + keptFiles := e.deleteMarkedFiles(files) + if len(results) > 0 { + results[len(results)-1].OutputFiles = keptFiles + } return results, nil } -// deleteMarkedFiles 删除标记为 Deletable 的文件，返回保留的文件列表 +// deleteMarkedFiles 删除标记为 Deletable 或 Metadata["uploaded"] 的文件，返回保留的文件列表 func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo { var kept []FileInfo + deleteAll := false // 标记是否为 deleteAll 模式 + deleteAfter := false // 标记是否为 deleteAfter 模式（仅删除已上传文件） + + // 获取输出路径用于计算相对路径（清理 DB 上传标记） + cfg := configs.GetCurrentConfig() + for _, f := range files { - if f.Deletable { + // 检查删除模式（CloudUploadStage 标记） + if f.Metadata != nil { + if da, ok := f.Metadata["delete_all"].(bool); ok && da { + deleteAll = true + } + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + deleteAfter = true + } + } + + // 判断是否应删除：Deletable 标记 或 CloudUploadStage 标记的已上传文件 + shouldDelete := f.Deletable + if !shouldDelete && f.Metadata != nil { + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + shouldDelete = true + } + } + + if shouldDelete { if err := os.Remove(f.Path); err != nil { if !os.IsNotExist(err) { e.logger.Warnf("pipeline cleanup: 删除文件失败 %s: %v", f.Path, err) + kept = append(kept, f) // 删除失败，文件仍在磁盘，保留记录 } } else { e.logger.Infof("pipeline cleanup: 已删除 %s", f.Path) + // 清除 DB 中的上传标记 + if cfg != nil { + outPutPath, _ := filepath.Abs(cfg.OutPutPath) + absFilePath, _ := filepath.Abs(f.Path) + if relPath, relErr := filepath.Rel(outPutPath, absFilePath); relErr == nil { + if delErr := metadata.GetStore().Delete(context.Background(), metadata.NamespaceUploaded, filepath.ToSlash(relPath)); delErr != nil { + e.logger.Warnf("pipeline cleanup: 清除上传标记失败 %s: %v", f.Path, delErr) + } + } + } } } else { kept = append(kept, f) } } - // 兜底逻辑：清理无关联视频文件的 .ass 文件 - // 传入原始 files 列表，确保即使视频被删除也能扫描到对应目录 - kept = e.cleanupOrphanedAssFiles(files, kept) + // 清理无关联视频文件的孤立 .ass 文件 + // deleteAll 模式下删除所有文件（含 .ass），deleteAfter 模式下删除已上传视频后也应清理残留 .ass + if deleteAll || deleteAfter { + kept = e.cleanupOrphanedAssFiles(files, kept) + } return kept } @@ -183,7 +318,7 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File dirs := map[string]bool{} for _, f := range allFiles { ext := strings.ToLower(filepath.Ext(f.Path)) - if ext == ".flv" || ext == ".mp4" || ext == ".mkv" { + if ext == ".flv" || ext == ".mp4" || ext == ".mkv" || ext == ".ts" { dir := filepath.Dir(f.Path) base := strings.TrimSuffix(filepath.Base(f.Path), ext) videoBaseNames[filepath.Join(dir, base)] = true @@ -199,7 +334,8 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File } } - // 扫描目录中的 .ass 文件（包括未进入 Pipeline 的） + // 扫描目录中的 .ass 文件 + // 通过视频基名匹配判断 .ass 是否属于本任务，避免误删其他任务或用户放置的字幕 for dir := range dirs { entries, err := os.ReadDir(dir) if err != nil { @@ -216,12 +352,16 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File if knownAssFiles[assPath] { continue // 已在 Pipeline 中处理 } - // 检查磁盘上是否有同名视频文件 + // 通过基名匹配判断是否属于本任务的视频文件 + // （.ass 文件可能未被传入 Pipeline，不能依赖文件列表过滤） base := strings.TrimSuffix(entry.Name(), ".ass") key := filepath.Join(dir, base) - // 检查磁盘上实际存在的视频文件 + if !videoBaseNames[key] { + continue // 基名不匹配任何本任务视频，跳过 + } + // 检查磁盘上是否有同名视频文件 hasVideo := false - for _, ext := range []string{".flv", ".mp4", ".mkv"} { + for _, ext := range []string{".flv", ".mp4", ".mkv", ".ts"} { if _, err := os.Stat(key + ext); err == nil { hasVideo = true break @@ -230,10 +370,10 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File if !hasVideo { if err := os.Remove(assPath); err != nil { if !os.IsNotExist(err) { - e.logger.Warnf("pipeline cleanup: 删除磁盘孤立 .ass 文件失败 %s: %v", assPath, err) + e.logger.Warnf("pipeline cleanup: 删除孤立 .ass 文件失败 %s: %v", assPath, err) } } else { - e.logger.Infof("pipeline cleanup: 已删除磁盘孤立 .ass 文件 %s（无关联视频）", assPath) + e.logger.Infof("pipeline cleanup: 已删除孤立 .ass 文件 %s（无关联视频）", assPath) } } } @@ -244,6 +384,10 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File for _, f := range keptFiles { ext := strings.ToLower(filepath.Ext(f.Path)) if ext == ".ass" { + // 先检查 .ass 是否还在磁盘上（可能已被 deleteMarkedFiles 删除） + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + continue // 已被删除，不加入 kept + } dir := filepath.Dir(f.Path) base := strings.TrimSuffix(filepath.Base(f.Path), ".ass") key := filepath.Join(dir, base) @@ -399,6 +543,18 @@ func deduplicateFiles(files []FileInfo) []FileInfo { return result } +// backfillFileSizes 对 Size 为 0 的文件执行 os.Stat 回填大小 +// 阶段内构造 FileInfo 时可能未 stat，导致摘要通知漏掉这些文件 +func backfillFileSizes(files []FileInfo) { + for i := range files { + if files[i].Size == 0 { + if fi, err := os.Stat(files[i].Path); err == nil { + files[i].Size = fi.Size() + } + } + } +} + // getTimeNow 获取当前时间（可用于测试时 mock） var getTimeNow = func() time.Time { return time.Now() @@ -415,7 +571,7 @@ func (e *Executor) ExecuteAsync( bilisentry.GoWithContext(ctx.Ctx, func(goCtx context.Context) { // 更新上下文 ctx.Ctx = goCtx - results, err := e.Execute(ctx, config, initialFiles, onProgress) + results, err := e.Execute(ctx, config, initialFiles, 0, onProgress, nil) if onComplete != nil { onComplete(results, err) } diff --git a/src/pipeline/executor_test.go b/src/pipeline/executor_test.go index b06d808..1889eab 100644 --- a/src/pipeline/executor_test.go +++ b/src/pipeline/executor_test.go @@ -86,7 +86,7 @@ func TestExecute_AllStagesSucceed_AllDeletableFilesRemoved(t *testing.T) { results, err := executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil, nil) // 验证：无错误 assert.NoError(t, err) @@ -142,7 +142,7 @@ func TestExecute_StageFailed_AllFilesPreserved(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil, nil) // 验证：返回错误 assert.Error(t, err) @@ -199,7 +199,7 @@ func TestExecute_Cancelled_AllFilesPreserved(t *testing.T) { _, _ = executor.Execute(ctx, config, []FileInfo{ {Path: originalFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil, nil) // 验证：stage2 返回成功，但下一轮循环检测到 context 已取消 // 注意：stage2 本身成功了，所以 deleteMarkedFiles 会被调用 @@ -248,7 +248,7 @@ func TestExecute_PassthroughPreservesDeletable(t *testing.T) { results, err := executor.Execute(ctx, config, []FileInfo{ {Path: file1, Type: FileTypeVideo}, {Path: file2, Type: FileTypeOther, Deletable: true}, - }, nil) + }, 0, nil, nil) assert.NoError(t, err) assert.Len(t, results, 1) @@ -304,7 +304,7 @@ func TestExecute_UploadCancelledDuringUpload_FilePreserved(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: localFile, Type: FileTypeVideo}, - }, nil) + }, 0, nil, nil) // 验证：返回错误 assert.Error(t, err) @@ -363,7 +363,7 @@ func TestExecute_UploadFailsMiddleOfFileList_NoneDeleted(t *testing.T) { {Path: file1, Type: FileTypeVideo}, {Path: file2, Type: FileTypeVideo}, {Path: file3, Type: FileTypeVideo}, - }, nil) + }, 0, nil, nil) assert.NoError(t, err) @@ -423,7 +423,7 @@ func TestExecute_StaleDeletableFromDB_ClearedOnFailure(t *testing.T) { _, err := executor.Execute(ctx, config, []FileInfo{ {Path: file1, Type: FileTypeVideo, Deletable: true}, // 旧标记 {Path: file2, Type: FileTypeVideo, Deletable: true}, // 旧标记 - }, nil) + }, 0, nil, nil) assert.NoError(t, err) diff --git a/src/pipeline/manager.go b/src/pipeline/manager.go index da7880d..50d4395 100644 --- a/src/pipeline/manager.go +++ b/src/pipeline/manager.go @@ -3,14 +3,21 @@ package pipeline import ( "context" "fmt" + "os" + "path/filepath" + "regexp" + "strings" "sync" "time" + "github.com/bililive-go/bililive-go/src/configs" "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" + "github.com/bililive-go/bililive-go/src/notify" "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry" + "github.com/bililive-go/bililive-go/src/types" "github.com/sirupsen/logrus" ) @@ -23,6 +30,7 @@ type Manager struct { executor *Executor config *ManagerConfig runningTasks map[int64]context.CancelFunc + taskCallbacks map[int64]func(*PipelineTask) // 按 task.ID 索引的完成回调（不经过持久化） mu sync.RWMutex wg sync.WaitGroup eventDispatch events.Dispatcher @@ -64,6 +72,7 @@ func NewManager(ctx context.Context, store Store, config *ManagerConfig, dispatc executor: NewExecutor(logrus.StandardLogger()), config: config, runningTasks: make(map[int64]context.CancelFunc), + taskCallbacks: make(map[int64]func(*PipelineTask)), eventDispatch: dispatcher, } @@ -201,11 +210,21 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { WorkDir: "", // 后续可以从配置获取 } - // 执行管道 + // 执行管道（从 startStage 开始，支持从指定阶段重试） + startStage := task.CurrentStage + + // 保留目标阶段之前的结果（用于合并） + var preservedResults []StageResult + if startStage > 0 && len(task.StageResults) >= startStage { + preservedResults = make([]StageResult, startStage) + copy(preservedResults, task.StageResults[:startStage]) + } + results, err := m.executor.Execute( pipelineCtx, task.PipelineConfig, task.CurrentFiles, + startStage, func(stageIndex int, stageName string, status StageStatus) { // 更新任务进度 task.CurrentStage = stageIndex @@ -215,10 +234,26 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { } m.broadcastTaskUpdate(task) }, + // 阶段完成后持久化结果，确保崩溃恢复时阶段记录完整 + func(stageIndex int, result StageResult) { + task.CurrentFiles = result.OutputFiles + // 追加/更新 StageResults，让崩溃恢复后 preservedResults 合并条件成立 + for len(task.StageResults) <= stageIndex { + task.StageResults = append(task.StageResults, StageResult{}) + } + task.StageResults[stageIndex] = result + if err := m.store.UpdateTask(ctx, task); err != nil { + logrus.WithError(err).Warn("failed to persist stage result") + } + }, ) - // 保存阶段结果 - task.StageResults = results + // 合并：保留之前的结果 + 新执行的结果 + if preservedResults != nil { + task.StageResults = append(preservedResults, results...) + } else { + task.StageResults = results + } if err != nil { if ctx.Err() == context.Canceled { @@ -238,6 +273,9 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { logrus.WithField("task_id", task.ID).Info("pipeline task completed successfully") } + // 保存最后阶段输出的快照（清理前），用于回调提取上传文件详情 + task.LastStageFiles = pipelineCtx.LastStageFiles + // 更新任务状态 if err := m.store.UpdateTask(m.ctx, task); err != nil { logrus.WithError(err).Error("failed to update pipeline task status after execution") @@ -245,6 +283,16 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) { // 广播任务状态变化 m.broadcastTaskUpdate(task) + + // 原子地取出并删除回调，确保每个任务的回调至多触发一次 + // 避免 executeTask 和 CancelTask 并发时双发回调 + m.mu.Lock() + callback := m.taskCallbacks[task.ID] + delete(m.taskCallbacks, task.ID) + m.mu.Unlock() + if callback != nil { + callback(task) + } } // broadcastTaskUpdate 广播任务更新事件 @@ -254,11 +302,36 @@ func (m *Manager) broadcastTaskUpdate(task *PipelineTask) { } } +// FileInfoToDetails 将 FileInfo 切片转换为 RecordingFileDetail 切片 +// 使用 FileInfo 中预存的 Size，无需再次 stat 文件 +// 仅包含视频和其他类型文件，排除封面（封面不在录制摘要中展示） +func FileInfoToDetails(files []FileInfo) []notify.RecordingFileDetail { + details := make([]notify.RecordingFileDetail, 0, len(files)) + for _, f := range files { + if f.Size > 0 && f.Type != FileTypeCover { + details = append(details, notify.RecordingFileDetail{ + Name: filepath.Base(f.Path), + Size: f.Size, + }) + } + } + return details +} + // EnqueueTask 添加任务到队列 -func (m *Manager) EnqueueTask(task *PipelineTask) error { +// onTaskComplete: 任务完成回调（可选），在任务对调度器可见前注册 +func (m *Manager) EnqueueTask(task *PipelineTask, onTaskComplete func(*PipelineTask)) error { + // 持锁覆盖 CreateTask + 回调注册，阻塞 pollLoop 的 scheduleNextTasks（需 RLock） + // 确保任务对调度器可见时回调已就绪 + m.mu.Lock() if err := m.store.CreateTask(m.ctx, task); err != nil { + m.mu.Unlock() return fmt.Errorf("failed to create pipeline task: %w", err) } + if onTaskComplete != nil { + m.taskCallbacks[task.ID] = onTaskComplete + } + m.mu.Unlock() logrus.WithFields(logrus.Fields{ "task_id": task.ID, @@ -277,10 +350,12 @@ func (m *Manager) EnqueueTask(task *PipelineTask) error { // EnqueueRecordingTask 创建并入队录制完成后的处理任务 // 这是 recorder.go 调用的主要入口 +// onTaskComplete: 任务完成回调（可选），用于通知 recorder 任务结束 func (m *Manager) EnqueueRecordingTask( info *live.Info, pipelineConfig *PipelineConfig, outputFiles []string, + onTaskComplete func(task *PipelineTask), ) error { // 构建文件信息列表 files := make([]FileInfo, len(outputFiles)) @@ -291,7 +366,8 @@ func (m *Manager) EnqueueRecordingTask( // 创建任务 task := NewPipelineTask(NewRecordInfo(info), pipelineConfig, files) - return m.EnqueueTask(task) + // 入队并注册回调（EnqueueTask 内部在异步调度前完成注册，避免竞态窗口） + return m.EnqueueTask(task, onTaskComplete) } // CancelTask 取消任务 @@ -316,6 +392,15 @@ func (m *Manager) CancelTask(taskID int64) error { return err } m.broadcastTaskUpdate(task) + + // 原子地取出并删除回调，避免与 executeTask 并发时双发 + m.mu.Lock() + callback := m.taskCallbacks[taskID] + delete(m.taskCallbacks, taskID) + m.mu.Unlock() + if callback != nil { + callback(task) + } } return nil @@ -332,8 +417,16 @@ func (m *Manager) RetryTask(taskID int64) error { return fmt.Errorf("task cannot be retried") } - if task.Status != PipelineStatusFailed && task.Status != PipelineStatusCancelled { - return fmt.Errorf("only failed or cancelled tasks can be retried") + // 允许 Failed/Cancelled/Completed 状态 + if task.Status != PipelineStatusFailed && task.Status != PipelineStatusCancelled && task.Status != PipelineStatusCompleted { + return fmt.Errorf("only failed, cancelled or completed tasks can be retried") + } + + // 校验初始文件存在（Completed 任务的文件可能已被 deleteMarkedFiles 清理） + for _, f := range task.InitialFiles { + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + return fmt.Errorf("initial file not found, cannot retry (may have been deleted after upload): %s", f.Path) + } } // 重置任务状态 @@ -342,6 +435,7 @@ func (m *Manager) RetryTask(taskID int64) error { task.CompletedAt = nil task.ErrorMessage = "" task.CurrentStage = 0 + task.CurrentFiles = task.InitialFiles // 重置为初始文件，从头开始重试 task.StageResults = nil task.Progress = 0 @@ -357,6 +451,122 @@ func (m *Manager) RetryTask(taskID int64) error { return nil } +// RetryTaskFromStage 从指定阶段开始重试任务 +// 允许 Failed/Cancelled/Completed 状态的任务 +// stageName: 要从哪个阶段开始重试（如 "cloud_upload"） +func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error { + task, err := m.store.GetTask(m.ctx, taskID) + if err != nil { + return err + } + + if !task.CanRetry { + return fmt.Errorf("task cannot be retried") + } + + // 允许 Failed/Cancelled/Completed 状态 + if task.Status != PipelineStatusFailed && + task.Status != PipelineStatusCancelled && + task.Status != PipelineStatusCompleted { + return fmt.Errorf("only failed, cancelled or completed tasks can be retried") + } + + // 找到目标阶段在启用阶段中的索引 + targetStageIndex := -1 + stageIndex := 0 + for _, stageCfg := range task.PipelineConfig.Stages { + if !stageCfg.IsEnabled() { + continue + } + if stageCfg.Name == stageName { + targetStageIndex = stageIndex + break + } + stageIndex++ + } + if targetStageIndex == -1 { + return fmt.Errorf("stage %s not found in pipeline", stageName) + } + + // 校验：目标阶段之前的所有阶段必须已完成 + if targetStageIndex > 0 { + for i := 0; i < targetStageIndex; i++ { + if i >= len(task.StageResults) { + return fmt.Errorf("stage %d has not been executed, cannot retry from stage %s", i, stageName) + } + if task.StageResults[i].Status != StageStatusCompleted { + return fmt.Errorf("stage %d (%s) is not completed (status: %s), cannot retry from stage %s", + i, task.StageResults[i].StageName, task.StageResults[i].Status, stageName) + } + } + } + + // 获取目标阶段的输入文件 + var inputFiles []FileInfo + if targetStageIndex < len(task.StageResults) { + // 从已有的 StageResult 中取输入文件 + inputFiles = task.StageResults[targetStageIndex].InputFiles + } else if targetStageIndex > 0 && targetStageIndex-1 < len(task.StageResults) { + // 使用上一阶段的输出文件 + inputFiles = task.StageResults[targetStageIndex-1].OutputFiles + } else if targetStageIndex == 0 { + // 从头开始，使用初始文件 + inputFiles = task.InitialFiles + } + + // 过滤已清理的中间文件，保留仍在磁盘上的文件 + // deleteMarkedFiles 仅在管道全部成功后执行，失败/取消时文件仍在磁盘但带有标记 + // 因此需要结合磁盘实际存在性判断：标记但仍在 → 保留（管道未清理），标记且已删 → 跳过 + var activeFiles []FileInfo + for _, f := range inputFiles { + isDeletable := f.Deletable + isUploaded := false + if f.Metadata != nil { + if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded { + isUploaded = true + } + } + + if _, err := os.Stat(f.Path); os.IsNotExist(err) { + if isDeletable || isUploaded { + continue // 已被 deleteMarkedFiles 正常清理，跳过 + } + return fmt.Errorf("input file not found, cannot retry: %s", f.Path) + } + activeFiles = append(activeFiles, f) + } + inputFiles = activeFiles + + // 校验输入文件不为空 + if len(inputFiles) == 0 { + return fmt.Errorf("no input files available for stage %s, cannot retry", stageName) + } + + // 重置任务状态 + task.Status = PipelineStatusPending + task.StartedAt = nil + task.CompletedAt = nil + task.ErrorMessage = "" + task.CurrentStage = targetStageIndex + task.CurrentFiles = inputFiles + // 保留目标阶段之前的结果，清除目标阶段及之后的结果 + if targetStageIndex < len(task.StageResults) { + task.StageResults = task.StageResults[:targetStageIndex] + } + task.UpdateProgress() + + if err := m.store.UpdateTask(m.ctx, task); err != nil { + return err + } + + m.broadcastTaskUpdate(task) + + // 立即尝试调度 + bilisentry.Go(func() { m.scheduleNextTasks() }) + + return nil +} + // GetTask 获取任务详情 func (m *Manager) GetTask(taskID int64) (*PipelineTask, error) { return m.store.GetTask(m.ctx, taskID) @@ -432,6 +642,112 @@ type ManagerStats struct { CancelledCount int `json:"cancelled_count"` } +// EnqueueUploadTask 创建手动上传的 Pipeline 任务并入队 +// 行为与自动上传（CloudUploadStage）完全一致：使用相同的路径模板、配置选项 +// 每个文件创建一个独立的单阶段 cloud_upload 任务 +// absPaths: 已校验的绝对路径列表（调用方应先通过 getSafePath 校验） +func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped []string, taskIDs []int64, err error) { + config := configs.GetCurrentConfig() + if config == nil { + return 0, nil, nil, fmt.Errorf("配置未初始化") + } + + cu := config.OnRecordFinished.CloudUpload + if !cu.Enable { + return 0, nil, nil, fmt.Errorf("云上传功能未启用，请先在设置中开启") + } + if cu.StorageName == "" { + return 0, nil, nil, fmt.Errorf("未配置存储名称") + } + + // 构建单阶段 cloud_upload PipelineConfig，与 ConvertLegacyConfig 一致 + uploadConfig := &PipelineConfig{ + Stages: []StageConfig{ + { + Name: StageNameCloudUpload, + Options: map[string]any{ + OptionStorage: cu.StorageName, + OptionPathTemplate: cu.UploadPathTmpl, + OptionDeleteAfter: cu.DeleteAfterUpload, + OptionDeleteAllAfter: cu.DeleteAllAfterUpload, + OptionUploadTiming: "after_process", // 手动上传无后续处理阶段，始终允许删除标记（不继承全局 upload_timing） + OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)}, + OptionUploadSubtitles: cu.UploadSubtitles, + }, + }, + }, + } + + outputPath, _ := filepath.Abs(config.OutPutPath) + skipped = []string{} + + for _, absPath := range absPaths { + // 检查文件存在 + info, statErr := os.Stat(absPath) + if statErr != nil { + skipped = append(skipped, filepath.Base(absPath)+" - 文件不存在或无法访问") + continue + } + if info.IsDir() { + skipped = append(skipped, filepath.Base(absPath)+" - 不支持上传文件夹") + continue + } + + // 从相对路径推断 RecordInfo（用于上传路径模板渲染） + relPath, _ := filepath.Rel(outputPath, absPath) + platform, hostName := inferUploadPathInfo(relPath) + + recordInfo := RecordInfo{ + LiveID: types.LiveID(ManualUploadLiveID), + Platform: platform, // 用于上传路径模板：{{ .Platform }} + HostName: hostName, // 用于上传路径模板：{{ .HostName }} + RoomName: "", // 手动上传无房间名，不影响路径模板 + DisplayName: filepath.Base(absPath), // 任务列表展示文件名 + StartTime: info.ModTime(), + } + + files := []FileInfo{NewVideoFileInfo(absPath)} + task := NewPipelineTask(recordInfo, uploadConfig, files) + + if enqueueErr := m.EnqueueTask(task, nil); enqueueErr != nil { + skipped = append(skipped, filepath.Base(absPath)+" - 入队失败: "+enqueueErr.Error()) + continue + } + + enqueued++ + taskIDs = append(taskIDs, task.ID) + } + + return enqueued, skipped, taskIDs, nil +} + +// bracketPattern 匹配文件名中的 [xxx] 模式 +var bracketPattern = regexp.MustCompile(`\[([^\]]*)\]`) + +// inferUploadPathInfo 从文件相对路径推断平台和主播名 +// 两级策略：优先从文件名解析（[date][HostName][RoomName].flv），回退到目录结构推断 +func inferUploadPathInfo(relPath string) (platform, hostName string) { + slashPath := filepath.ToSlash(relPath) + + // 策略 1：从文件名解析 HostName（文件名格式通常是 [date][HostName][RoomName].flv） + fileName := filepath.Base(slashPath) + brackets := bracketPattern.FindAllStringSubmatch(fileName, -1) + if len(brackets) >= 2 { + hostName = brackets[1][1] // 第二个方括号通常是主播名 + } + + // 策略 2：从目录结构推断 Platform，以及作为 HostName 的兜底 + parts := strings.Split(slashPath, "/") + if len(parts) >= 2 { + platform = parts[0] + if hostName == "" { + hostName = parts[1] + } + } + + return platform, hostName +} + // GetManager 从实例获取管道管理器 func GetManager(inst *instance.Instance) *Manager { if inst == nil || inst.PipelineManager == nil { diff --git a/src/pipeline/stages/burn_subtitles.go b/src/pipeline/stages/burn_subtitles.go index fa2f8db..5fce8f6 100644 --- a/src/pipeline/stages/burn_subtitles.go +++ b/src/pipeline/stages/burn_subtitles.go @@ -83,6 +83,13 @@ func (s *BurnSubtitlesStage) Execute(ctx *pipeline.PipelineContext, input []pipe continue } + // 跳过已标记待删除的文件（如 ConvertMp4 阶段标记的原始 FLV）， + // 但仍将其透传到 output，保证 Executor 的延迟删除机制正常工作。 + if file.Deletable { + output = append(output, file) + continue + } + // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) diff --git a/src/pipeline/stages/convert_mp4.go b/src/pipeline/stages/convert_mp4.go index 53daedb..73457e4 100644 --- a/src/pipeline/stages/convert_mp4.go +++ b/src/pipeline/stages/convert_mp4.go @@ -120,6 +120,12 @@ func (s *ConvertMp4Stage) Execute(ctx *pipeline.PipelineContext, input []pipelin return nil, fmt.Errorf("failed to create stdout pipe: %w", err) } + stderr, err := cmd.StderrPipe() + if err != nil { + s.logs += fmt.Sprintf("创建错误管道失败: %s\n", err.Error()) + return nil, fmt.Errorf("failed to create stderr pipe: %w", err) + } + if err := cmd.Start(); err != nil { s.logs += fmt.Sprintf("启动 ffmpeg 失败: %s\n", err.Error()) return nil, fmt.Errorf("failed to start ffmpeg: %w", err) @@ -130,10 +136,14 @@ func (s *ConvertMp4Stage) Execute(ctx *pipeline.PipelineContext, input []pipelin s.parseProgress(goCtx, stdout, duration) }) + // 读取 stderr 防止缓冲区满导致 ffmpeg 挂起 + stderrBytes, _ := io.ReadAll(stderr) + // 等待命令完成 if err := cmd.Wait(); err != nil { os.Remove(tempFile) s.logs += fmt.Sprintf("ffmpeg 转换失败: %s - %s\n", file.Path, err.Error()) + s.logs += fmt.Sprintf("ffmpeg stderr: %s\n", string(stderrBytes)) return nil, fmt.Errorf("ffmpeg conversion failed for %s: %w", file.Path, err) } diff --git a/src/pipeline/stages/custom_command.go b/src/pipeline/stages/custom_command.go index c5ddc10..cdcb316 100644 --- a/src/pipeline/stages/custom_command.go +++ b/src/pipeline/stages/custom_command.go @@ -51,6 +51,14 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe var output []pipeline.FileInfo for _, file := range input { + // 跳过中间产物（如 ConvertMp4 标记的原始 FLV），避免重复执行命令。 + // 但 delete_all_after_upload 模式会将所有文件（含最终成品）标记为 Deletable， + // 此类文件仍需执行自定义命令，不能跳过。 + if file.Deletable && !isDeleteAllMarked(file) { + output = append(output, file) + continue + } + // 渲染命令模板 cmdStr, err := s.renderCommand(ctx, file) if err != nil { @@ -77,6 +85,16 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe return output, nil } +// isDeleteAllMarked 检查文件是否被 delete_all_after_upload 模式标记 +// 此类文件虽为 Deletable，但属于最终成品，仍需执行自定义命令 +func isDeleteAllMarked(file pipeline.FileInfo) bool { + if file.Metadata == nil { + return false + } + da, ok := file.Metadata["delete_all"].(bool) + return ok && da +} + // renderCommand 渲染命令模板 func (s *CustomCommandStage) renderCommand(ctx *pipeline.PipelineContext, file pipeline.FileInfo) (string, error) { // 构建模板数据 diff --git a/src/pipeline/stages/extract_cover.go b/src/pipeline/stages/extract_cover.go index 5547d78..1d54e29 100644 --- a/src/pipeline/stages/extract_cover.go +++ b/src/pipeline/stages/extract_cover.go @@ -1,17 +1,16 @@ package stages import ( - "bytes" "fmt" "os" "path" "path/filepath" "strings" - "text/template" "time" "github.com/bililive-go/bililive-go/src/configs" "github.com/bililive-go/bililive-go/src/pipeline" + "github.com/bililive-go/bililive-go/src/pkg/metadata" "github.com/bililive-go/bililive-go/src/pkg/openlist" "github.com/bililive-go/bililive-go/src/tools" ) @@ -52,6 +51,12 @@ func (s *ExtractCoverStage) Execute(ctx *pipeline.PipelineContext, input []pipel continue } + // 跳过已标记待删除的文件（如 ConvertMp4 阶段标记的原始 FLV）， + // 避免重复提取封面。 + if file.Deletable { + continue + } + // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) @@ -92,27 +97,29 @@ func (s *ExtractCoverStage) GetLogs() string { // CloudUploadStage 云上传阶段 type CloudUploadStage struct { - config pipeline.StageConfig - storageName string - pathTemplate string - deleteAfter bool // 仅删除已上传的文件 - deleteAllAfter bool // 删除全部文件（含中间产物） - uploadTiming string // immediate 或 after_process - fileTypes []string // 过滤的文件类型，空表示所有 - commands []string - logs string + config pipeline.StageConfig + storageName string + pathTemplate string + deleteAfter bool // 仅删除已上传的文件 + deleteAllAfter bool // 删除全部文件（含中间产物） + uploadTiming string // immediate 或 after_process + fileTypes []string // 过滤的文件类型，空表示所有 + uploadSubtitles bool // 是否上传关联的 .ass 弹幕字幕文件 + commands []string + logs string } // NewCloudUploadStage 创建云上传阶段工厂 func NewCloudUploadStage(config pipeline.StageConfig) (pipeline.Stage, error) { return &CloudUploadStage{ - config: config, - storageName: config.GetStringOption(pipeline.OptionStorage, ""), - pathTemplate: config.GetStringOption(pipeline.OptionPathTemplate, ""), - deleteAfter: config.GetBoolOption(pipeline.OptionDeleteAfter, false), - deleteAllAfter: config.GetBoolOption(pipeline.OptionDeleteAllAfter, false), - uploadTiming: config.GetStringOption(pipeline.OptionUploadTiming, ""), - fileTypes: config.GetStringSliceOption(pipeline.OptionFileTypes), + config: config, + storageName: config.GetStringOption(pipeline.OptionStorage, ""), + pathTemplate: config.GetStringOption(pipeline.OptionPathTemplate, ""), + deleteAfter: config.GetBoolOption(pipeline.OptionDeleteAfter, false), + deleteAllAfter: config.GetBoolOption(pipeline.OptionDeleteAllAfter, false), + uploadTiming: config.GetStringOption(pipeline.OptionUploadTiming, ""), + fileTypes: config.GetStringSliceOption(pipeline.OptionFileTypes), + uploadSubtitles: config.GetBoolOption(pipeline.OptionUploadSubtitles, false), }, nil } @@ -134,12 +141,16 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 获取 OpenList 管理器 mgr := openlist.GetGlobalManager() if mgr == nil { - s.logs = "OpenList 管理器未初始化，跳过上传\n" - return input, nil + s.logs = "OpenList 管理器未初始化\n" + return input, fmt.Errorf("OpenList 管理器未初始化") } // 获取配置并创建客户端 config := configs.GetCurrentConfig() + if config == nil { + s.logs = "配置未初始化\n" + return input, fmt.Errorf("配置未初始化") + } client, err := mgr.GetClient(ctx.Ctx, config.OpenList.Token, config.OpenList.Username, config.OpenList.Password) if err != nil { @@ -149,8 +160,11 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli // 如果 token 变化了（通过用户名密码登录获取了新 token），自动写回配置 if newToken := client.GetCurrentToken(); newToken != "" && newToken != config.OpenList.Token { - config.OpenList.Token = newToken - if err := config.Marshal(); err != nil { + _, err := configs.UpdateWithRetry(func(c *configs.Config) error { + c.OpenList.Token = newToken + return nil + }, 3, 10*time.Millisecond) + if err != nil { ctx.Logger.Warnf("保存 OpenList token 到配置文件失败: %v", err) } else { ctx.Logger.Info("OpenList token 已自动更新到配置文件") @@ -159,54 +173,111 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli var output []pipeline.FileInfo var uploadFailed bool + usedPaths := map[string]bool{} // 检测远程路径冲突 + var failureDetails []string // 收集每条失败详情，用于返回给前端 - // 清除所有输入文件的 Deletable 标记（防止数据库中残留的旧标记影响本次判断） - for i := range input { - input[i].Deletable = false - } - - // 只上传第一个视频文件和封面文件，跳过中间产物 - // Pipeline 阶段输出顺序保证：最终视频在最前面，中间文件在后面 - videoUploaded := false + // 上传所有非 Deletable 的视频文件和封面文件 + // Deletable 检查已能正确区分中间产物（如 ConvertMp4 标记的原始 FLV） + // 与最终视频（如录播姬生成的多分段 _PART 文件），无需额外限制 uploadedIndices := map[int]bool{} // 记录实际上传的文件在 output 中的索引 + // 上传弹幕字幕：扫描 input 中的视频文件，从磁盘发现关联的 .ass 文件并加入 input + if s.uploadSubtitles { + assAdded := map[string]bool{} + var discovered []pipeline.FileInfo + for _, f := range input { + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { + continue + } + assPath := strings.TrimSuffix(f.Path, ext) + ".ass" + if assAdded[assPath] { + continue + } + if info, err := os.Stat(assPath); err == nil && info.Size() > 0 { + // 检查 input 和 discovered 中是否已存在 + alreadyExists := false + for _, of := range input { + if of.Path == assPath { + alreadyExists = true + break + } + } + if !alreadyExists { + for _, of := range discovered { + if of.Path == assPath { + alreadyExists = true + break + } + } + } + if !alreadyExists { + discovered = append(discovered, pipeline.FileInfo{ + Path: assPath, + Size: info.Size(), + Type: pipeline.FileTypeOther, + }) + assAdded[assPath] = true + ctx.Logger.Infof("发现关联弹幕字幕文件: %s", assPath) + } + } + } + if len(discovered) > 0 { + input = append(input, discovered...) + } + } + for _, file := range input { // 跳过标记为待删除的中间文件（由其他阶段产出，不应上传） + // 例外：uploadSubtitles 时放行 Deletable 的 .ass（如 burn_delete_ass 标记的字幕） if file.Deletable { - output = append(output, file) - continue - } - - // 文件类型过滤 - if len(s.fileTypes) > 0 && !s.matchFileType(file.Type) { - output = append(output, file) - continue - } - - // 视频文件：只上传第一个（最终视频），跳过后续中间视频 - if file.Type == pipeline.FileTypeVideo { - if videoUploaded { + if !s.uploadSubtitles || strings.ToLower(filepath.Ext(file.Path)) != ".ass" { + output = append(output, file) + continue + } + } + + // 文件类型过滤（uploadSubtitles 时放行 .ass 字幕文件） + if len(s.fileTypes) > 0 && !s.matchFileType(file.Type) { + if !s.uploadSubtitles || strings.ToLower(filepath.Ext(file.Path)) != ".ass" { output = append(output, file) continue } - videoUploaded = true } // 检查文件是否存在 if _, err := os.Stat(file.Path); os.IsNotExist(err) { - s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path) + errMsg := fmt.Sprintf("文件不存在: %s", file.Path) + s.logs += errMsg + "\n" + failureDetails = append(failureDetails, errMsg) + output = append(output, file) + uploadFailed = true continue } // 渲染目标路径 targetPath := s.renderTargetPath(ctx, file) if targetPath == "" { - s.logs += fmt.Sprintf("无法生成目标路径: %s\n", file.Path) + errMsg := fmt.Sprintf("无法生成目标路径: %s", file.Path) + s.logs += errMsg + "\n" + failureDetails = append(failureDetails, errMsg) output = append(output, file) uploadFailed = true continue } + // 检测远程路径冲突（多分段视频或视频+封面使用相同模板时可能产生同路径） + // 冲突时自动追加原始文件名以区分 + if usedPaths[targetPath] { + // 使用 path.* 而非 filepath.*，因为 targetPath 是远程路径，始终用正斜杠 + dir := path.Dir(targetPath) + ext := filepath.Ext(targetPath) + base := strings.TrimSuffix(path.Base(targetPath), ext) + targetPath = path.Join(dir, base+"_"+filepath.Base(file.Path)) + ctx.Logger.Infof("检测到远程路径冲突，自动追加文件名: %s", targetPath) + } + usedPaths[targetPath] = true + ctx.Logger.Infof("上传文件: %s -> %s", file.Path, targetPath) s.commands = append(s.commands, fmt.Sprintf("upload %s to %s/%s", file.Path, s.storageName, targetPath)) @@ -218,6 +289,7 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli } // 确保存储名和路径之间没有双斜杠 fullRemotePath := "/" + s.storageName + remotePath + fullRemotePath = strings.ReplaceAll(fullRemotePath, "//", "/") // 确保远程目录存在（递归创建） // 使用 path.Dir 而非 filepath.Dir，因为远程路径始终用正斜杠 @@ -236,7 +308,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli }) if err != nil { - s.logs += fmt.Sprintf("上传失败: %s -> %s/%s - %s\n", fileName, s.storageName, targetPath, err.Error()) + errMsg := fmt.Sprintf("上传失败: %s -> %s/%s - %s", fileName, s.storageName, targetPath, err.Error()) + s.logs += errMsg + "\n" + failureDetails = append(failureDetails, errMsg) ctx.Logger.Errorf("上传失败: %s - %v", file.Path, err) // 上传失败，保留文件 output = append(output, file) @@ -246,37 +320,153 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli s.logs += fmt.Sprintf("上传成功: %s -> %s/%s\n", fileName, s.storageName, targetPath) ctx.Logger.Infof("文件上传成功: %s -> %s/%s", fileName, s.storageName, targetPath) + // 标记已上传（立即设置，不依赖后续删除逻辑块，确保 immediate 模式下也有标记） + if file.Metadata == nil { + file.Metadata = map[string]any{} + } + file.Metadata["uploaded"] = true + // 记录上传路径，用于后续阶段替换文件后重新标记 + if ctx.UploadedPaths != nil { + ctx.UploadedPaths[file.Path] = true + } uploadedIndices[len(output)] = true // 记录上传成功的文件索引 output = append(output, file) + + // 持久化上传标记（统一用正斜杠存储，与 URL 路径一致） + if cfg := configs.GetCurrentConfig(); cfg != nil { + outPutPath, _ := filepath.Abs(cfg.OutPutPath) + absFilePath, _ := filepath.Abs(file.Path) + if relPath, relErr := filepath.Rel(outPutPath, absFilePath); relErr == nil { + if markErr := metadata.GetStore().Set(ctx.Ctx, metadata.NamespaceUploaded, filepath.ToSlash(relPath), "1"); markErr != nil { + ctx.Logger.Warnf("保存上传标记失败: %v", markErr) + } + } + } + } + + // 为已上传的 Deletable 文件标记 Metadata["uploaded"]（如 burn_delete_ass 标记的 .ass） + // 这些文件在上传循环中被放行（uploadSubtitles），但 Deletable 跳过逻辑未标记 uploaded + for i, f := range output { + if uploadedIndices[i] && f.Deletable { + if f.Metadata == nil { + f.Metadata = map[string]any{} + } + f.Metadata["uploaded"] = true + output[i] = f + } } // 全部上传成功后，根据配置标记文件为可删除（由 Executor 在管道全部成功后统一删除） // 任意一个上传失败，则不标记任何文件，确保本地文件全部保留 + // 注意：不使用 Deletable 标记，避免后续阶段（如 CustomCommand）误跳过最终成品 + // 改用 Metadata["uploaded"] 标记，由 Executor 在清理时读取 if !uploadFailed && s.uploadTiming != "immediate" { if s.deleteAllAfter { - // 删除全部文件（含中间产物） + // 删除全部模式：查找视频文件关联的 .ass 字幕文件并加入 output + // （BurnSubtitlesStage 仅在 burn_delete_ass=true 时才将 .ass 加入 output， + // 但 delete_all 意图是删除所有文件，应包含字幕） + assAdded := map[string]bool{} + for _, f := range output { + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { + continue + } + assPath := strings.TrimSuffix(f.Path, ext) + ".ass" + if assAdded[assPath] { + continue + } + if _, err := os.Stat(assPath); err == nil { + // 检查 output 中是否已存在 + alreadyInOutput := false + for _, of := range output { + if of.Path == assPath { + alreadyInOutput = true + break + } + } + if !alreadyInOutput { + output = append(output, pipeline.FileInfo{ + Path: assPath, + Type: pipeline.FileTypeOther, + Deletable: true, // 标记为可删除，由 executor 统一清理 + }) + assAdded[assPath] = true + ctx.Logger.Infof("delete_all 模式：关联字幕文件 %s", assPath) + } + } + } + + // 删除全部文件（含中间产物）：用 Deletable 标记 + // 跳过已是 Deletable 的中间产物（如 BurnSubtitles 标记的源视频）， + // 避免 delete_all 标记覆盖其"中间产物"身份，导致 CustomCommandStage 误执行 for i := range output { + if output[i].Deletable { + // 中间产物：保留原始 Deletable 标记，不加 delete_all + continue + } output[i].Deletable = true + if output[i].Metadata == nil { + output[i].Metadata = map[string]any{} + } + output[i].Metadata["delete_all"] = true + // 为实际上传的文件标记 uploaded（供回调提取上传文件详情） + if uploadedIndices[i] { + output[i].Metadata["uploaded"] = true + } } s.logs += fmt.Sprintf("全部上传成功，已标记 %d 个文件待删除（含中间产物）\n", len(output)) ctx.Logger.Infof("全部上传成功，已标记 %d 个文件待删除（含中间产物）", len(output)) } else if s.deleteAfter { - // 仅删除已上传的文件 + // 仅删除已上传的文件：用 Metadata["uploaded"] 标记，不设 Deletable count := 0 for i := range output { if uploadedIndices[i] { - output[i].Deletable = true + if output[i].Metadata == nil { + output[i].Metadata = map[string]any{} + } + output[i].Metadata["uploaded"] = true count++ } } + // 已上传的视频文件会被删除，其关联的 .ass 字幕文件也应标记为可删除 + // 避免摘要将已删除的字幕文件误显为本地保留文件 + // 注意：这些 .ass 未被上传到云存储，不标记 uploaded，仅标记 Deletable + for i, f := range output { + if !uploadedIndices[i] { + continue // 仅处理已上传成功的视频文件 + } + ext := strings.ToLower(filepath.Ext(f.Path)) + if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" { + continue + } + assPath := strings.TrimSuffix(f.Path, ext) + ".ass" + if _, err := os.Stat(assPath); err == nil { + alreadyInOutput := false + for _, of := range output { + if of.Path == assPath { + alreadyInOutput = true + break + } + } + if !alreadyInOutput { + output = append(output, pipeline.FileInfo{ + Path: assPath, + Type: pipeline.FileTypeOther, + Deletable: true, // 标记为可删除，由 executor 统一清理 + }) + ctx.Logger.Infof("delete_uploaded 模式：关联字幕文件 %s", assPath) + } + } + } s.logs += fmt.Sprintf("全部上传成功，已标记 %d 个已上传文件待删除\n", count) ctx.Logger.Infof("全部上传成功，已标记 %d 个已上传文件待删除", count) } } // 任意上传失败，返回错误（让 Executor 将任务标记为 failed） + // 将每条失败详情拼入 error message，前端可直接展示具体原因 if uploadFailed { - return output, fmt.Errorf("部分文件上传失败，详见日志") + return output, fmt.Errorf("部分文件上传失败:\n%s", strings.Join(failureDetails, "\n")) } return output, nil @@ -294,29 +484,12 @@ func (s *CloudUploadStage) matchFileType(fileType pipeline.FileType) bool { // renderTargetPath 渲染目标路径 func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo) string { - if s.pathTemplate == "" { - // 默认路径：/录播归档/{平台}/{主播名}/{文件名} - return fmt.Sprintf("/录播归档/%s/%s/%s", - ctx.RecordInfo.Platform, - ctx.RecordInfo.HostName, - filepath.Base(file.Path), - ) - } - - // 获取扩展名 ext := filepath.Ext(file.Path) if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } - // 模板数据 - data := struct { - Platform string - HostName string - RoomName string - FileName string - Ext string - }{ + data := pipeline.UploadPathData{ Platform: ctx.RecordInfo.Platform, HostName: ctx.RecordInfo.HostName, RoomName: ctx.RecordInfo.RoomName, @@ -324,64 +497,13 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file Ext: ext, } - // 使用 Go 模板引擎 - funcMap := template.FuncMap{ - "date": func(format string, t time.Time) string { - return t.Format(format) - }, - "now": func() time.Time { - return time.Now() - }, - "trimSuffix": func(suffix, s string) string { - return strings.TrimSuffix(s, suffix) - }, - "ext": func(path string) string { - return filepath.Ext(path) - }, - "filenameFilter": func(s string) string { - // 过滤文件名中的非法字符 - replacer := strings.NewReplacer( - "/", "_", "\\", "_", ":", "_", "*", "_", - "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", - ) - return replacer.Replace(s) - }, - } - - tmpl, err := template.New("path").Funcs(funcMap).Parse(s.pathTemplate) + path, err := pipeline.RenderUploadPath(s.pathTemplate, data, func() time.Time { + return ctx.RecordInfo.StartTime + }) if err != nil { - // 模板解析失败，回退到简单替换 - return s.fallbackRender(data) + ctx.Logger.Errorf("%v", err) + return "" } - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, data); err != nil { - // 模板执行失败，回退到简单替换 - return s.fallbackRender(data) - } - - return buf.String() -} - -// fallbackRender 简单字符串替换（模板引擎失败时的回退方案） -func (s *CloudUploadStage) fallbackRender(data struct { - Platform string - HostName string - RoomName string - FileName string - Ext string -}) string { - path := s.pathTemplate - path = strings.ReplaceAll(path, "{{ .Platform }}", data.Platform) - path = strings.ReplaceAll(path, "{{.Platform}}", data.Platform) - path = strings.ReplaceAll(path, "{{ .HostName }}", data.HostName) - path = strings.ReplaceAll(path, "{{.HostName}}", data.HostName) - path = strings.ReplaceAll(path, "{{ .RoomName }}", data.RoomName) - path = strings.ReplaceAll(path, "{{.RoomName}}", data.RoomName) - path = strings.ReplaceAll(path, "{{ .FileName }}", data.FileName) - path = strings.ReplaceAll(path, "{{.FileName}}", data.FileName) - path = strings.ReplaceAll(path, "{{ .Ext }}", data.Ext) - path = strings.ReplaceAll(path, "{{.Ext}}", data.Ext) return path } diff --git a/src/pipeline/types.go b/src/pipeline/types.go index 6da83df..70e569d 100644 --- a/src/pipeline/types.go +++ b/src/pipeline/types.go @@ -4,6 +4,7 @@ package pipeline import ( "context" + "os" "time" "github.com/bililive-go/bililive-go/src/live" @@ -15,6 +16,9 @@ import ( // PipelineTaskUpdateEvent 管道任务更新事件 const PipelineTaskUpdateEvent events.EventType = "PipelineTaskUpdate" +// ManualUploadLiveID 手动上传任务的 LiveID 标识 +const ManualUploadLiveID = "manual-upload" + // FileType 文件类型 type FileType string @@ -30,18 +34,23 @@ const ( // FileInfo 表示管道中流转的文件信息 type FileInfo struct { Path string `json:"path"` // 文件绝对路径 + Size int64 `json:"size,omitempty"` // 文件大小（字节），在任务创建时记录 Type FileType `json:"type"` // 文件类型 SourcePath string `json:"source_path,omitempty"` // 来源文件路径（用于追踪转换链） Metadata map[string]any `json:"metadata,omitempty"` // 额外元数据 Deletable bool `json:"deletable,omitempty"` // 标记此文件在管道全部成功后可删除 } -// NewVideoFileInfo 创建视频文件信息 +// NewVideoFileInfo 创建视频文件信息，自动记录文件大小 func NewVideoFileInfo(path string) FileInfo { - return FileInfo{ + fi := FileInfo{ Path: path, Type: FileTypeVideo, } + if stat, err := os.Stat(path); err == nil { + fi.Size = stat.Size() + } + return fi } // NewCoverFileInfo 创建封面文件信息 @@ -55,11 +64,12 @@ func NewCoverFileInfo(path, sourcePath string) FileInfo { // RecordInfo 录制信息 type RecordInfo struct { - LiveID types.LiveID `json:"live_id"` - Platform string `json:"platform"` - HostName string `json:"host_name"` - RoomName string `json:"room_name"` - StartTime time.Time `json:"start_time"` + LiveID types.LiveID `json:"live_id"` + Platform string `json:"platform"` + HostName string `json:"host_name"` + RoomName string `json:"room_name"` + DisplayName string `json:"display_name,omitempty"` // 任务列表展示用，不影响路径渲染 + StartTime time.Time `json:"start_time"` } // NewRecordInfo 从 live.Info 创建录制信息 @@ -83,6 +93,15 @@ type PipelineContext struct { // FFmpegPath 是 ffmpeg 可执行文件的路径 FFmpegPath string + + // LastStageFiles 最后阶段输出的文件快照（清理前保存） + // 用于回调提取上传文件详情，因为 deleteMarkedFiles 会删除文件 + LastStageFiles []FileInfo + + // UploadedPaths 记录 cloud_upload 阶段成功上传的文件路径 + // 用于在所有阶段执行完毕后，为被后续阶段替换的文件重新标记 Metadata["uploaded"] + // （如 fix_flv 替换 .flv 后，新 FileInfo 丢失上传标记） + UploadedPaths map[string]bool } // Stage 管道阶段接口 @@ -242,6 +261,7 @@ type PipelineTask struct { CompletedAt *time.Time `json:"completed_at,omitempty"` ErrorMessage string `json:"error_message,omitempty"` CanRetry bool `json:"can_retry"` // 是否可以重试 + LastStageFiles []FileInfo `json:"-"` // 最后阶段输出文件（清理前快照，不持久化） } // NewPipelineTask 创建新的管道任务 diff --git a/src/pipeline/upload_utils.go b/src/pipeline/upload_utils.go new file mode 100644 index 0000000..d6c0b88 --- /dev/null +++ b/src/pipeline/upload_utils.go @@ -0,0 +1,67 @@ +package pipeline + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" + "text/template" + "time" +) + +// UploadPathData 上传路径模板数据 +type UploadPathData struct { + Platform string + HostName string + RoomName string + FileName string + Ext string +} + +// DefaultUploadPathFuncs 返回上传路径模板的默认函数集 +// nowFn 返回当前时间（可从 RecordInfo.StartTime 或 time.Now() 传入） +func DefaultUploadPathFuncs(nowFn func() time.Time) template.FuncMap { + return template.FuncMap{ + "date": func(format string, t time.Time) string { + return t.Format(format) + }, + "now": nowFn, + "trimSuffix": func(suffix, s string) string { + return strings.TrimSuffix(s, suffix) + }, + "ext": func(path string) string { + return filepath.Ext(path) + }, + "filenameFilter": func(s string) string { + replacer := strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", + "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", + ) + return replacer.Replace(s) + }, + } +} + +// RenderUploadPath 渲染上传目标路径 +// pathTemplate 为空时使用默认路径 /录播归档/{Platform}/{HostName}/{FileName} +// 返回空字符串表示模板解析失败 +func RenderUploadPath(pathTemplate string, data UploadPathData, nowFn func() time.Time) (string, error) { + if pathTemplate == "" { + path := fmt.Sprintf("/录播归档/%s/%s/%s", data.Platform, data.HostName, data.FileName) + return path, nil + } + + funcMap := DefaultUploadPathFuncs(nowFn) + + tmpl, err := template.New("path").Funcs(funcMap).Parse(pathTemplate) + if err != nil { + return "", fmt.Errorf("上传路径模板解析失败: %w", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("上传路径模板执行失败: %w", err) + } + + return buf.String(), nil +} diff --git a/src/pkg/metadata/store.go b/src/pkg/metadata/store.go index d1c377e..ddb7fe0 100644 --- a/src/pkg/metadata/store.go +++ b/src/pkg/metadata/store.go @@ -92,7 +92,7 @@ func Close() error { storeMu.Lock() defer storeMu.Unlock() - if globalStore == nil { + if globalStore == nil || globalStore.db == nil { return nil } @@ -103,6 +103,9 @@ func Close() error { // Get 从指定命名空间获取值 func (s *Store) Get(ctx context.Context, namespace, key string) (string, error) { + if s == nil { + return "", nil + } s.mu.RLock() defer s.mu.RUnlock() @@ -123,6 +126,9 @@ func (s *Store) Get(ctx context.Context, namespace, key string) (string, error) // Set 在指定命名空间设置值 func (s *Store) Set(ctx context.Context, namespace, key, value string) error { + if s == nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() @@ -142,6 +148,9 @@ func (s *Store) Set(ctx context.Context, namespace, key, value string) error { // Delete 从指定命名空间删除键 func (s *Store) Delete(ctx context.Context, namespace, key string) error { + if s == nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() @@ -157,6 +166,9 @@ func (s *Store) Delete(ctx context.Context, namespace, key string) error { // GetAll 获取指定命名空间的所有键值对 func (s *Store) GetAll(ctx context.Context, namespace string) (map[string]string, error) { + if s == nil { + return nil, nil + } s.mu.RLock() defer s.mu.RUnlock() @@ -183,6 +195,9 @@ func (s *Store) GetAll(ctx context.Context, namespace string) (map[string]string // DeleteNamespace 删除整个命名空间 func (s *Store) DeleteNamespace(ctx context.Context, namespace string) error { + if s == nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() @@ -206,6 +221,8 @@ const ( NamespaceUpdate = "update" // NamespaceMigration 数据库迁移状态 NamespaceMigration = "migration" + // NamespaceUploaded 已上传文件标记（key=相对路径, value="1"） + NamespaceUploaded = "uploaded" ) // 预定义的键常量 diff --git a/src/pkg/openlist/client.go b/src/pkg/openlist/client.go index 0c6ab14..59f8f5d 100644 --- a/src/pkg/openlist/client.go +++ b/src/pkg/openlist/client.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" "os" "strings" + "sync" ) // encodePath 对路径的每个段分别编码，保留 / 分隔符 @@ -24,6 +26,7 @@ func encodePath(path string) string { // Client OpenList API 客户端 type Client struct { baseURL string + mu sync.RWMutex // 保护 token/username/password 的并发访问 token string username string // 用于 token 刷新 password string // 用于 token 刷新 @@ -43,40 +46,75 @@ func NewClient(baseURL, token string) *Client { // SetToken 设置 API Token func (c *Client) SetToken(token string) { + c.mu.Lock() + defer c.mu.Unlock() c.token = token } -// GetToken 获取当前 API Token +// GetCurrentToken 获取当前 API Token func (c *Client) GetCurrentToken() string { + c.mu.RLock() + defer c.mu.RUnlock() return c.token } // SetCredentials 设置用户名密码（用于 token 自动刷新） func (c *Client) SetCredentials(username, password string) { + c.mu.Lock() + defer c.mu.Unlock() c.username = username c.password = password } +// getTokenForRequest 获取用于请求的 token 副本（内部使用） +func (c *Client) getTokenForRequest() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.token +} + // refreshToken 刷新 token func (c *Client) refreshToken(ctx context.Context) error { - if c.username == "" || c.password == "" { + c.mu.RLock() + username := c.username + password := c.password + c.mu.RUnlock() + + if username == "" || password == "" { return fmt.Errorf("无法刷新 token: 未配置用户名密码") } - newToken, err := c.GetToken(ctx, c.username, c.password) + newToken, err := c.GetToken(ctx, username, password) if err != nil { return fmt.Errorf("刷新 token 失败: %w", err) } - c.token = newToken + c.SetToken(newToken) return nil } -// isAuthError 判断是否为认证错误（HTTP 401/403） +// APIError 表示 OpenList API 返回的结构化错误 +// 用于在 isAuthError 中识别认证失败（code 401/403） +type APIError struct { + Code int + Message string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("API 错误 (code %d): %s", e.Code, e.Message) +} + +// isAuthError 判断是否为认证错误 +// 匹配 HTTP 状态码 401/403 或 OpenList JSON code 401/403 func isAuthError(err error) bool { if err == nil { return false } + // 检查结构化 API 错误 + var apiErr *APIError + if errors.As(err, &apiErr) { + return apiErr.Code == 401 || apiErr.Code == 403 + } + // 兜底：检查错误文本中的 HTTP 状态码 s := err.Error() - // 匹配 "HTTP 401" 或 "HTTP 403" 格式，避免误匹配其他包含 401/403 的字符串 return strings.Contains(s, "HTTP 401") || strings.Contains(s, "HTTP 403") } @@ -123,7 +161,7 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP return fmt.Errorf("创建请求失败: %w", err) } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Length", fmt.Sprintf("%d", totalSize)) // 对路径的每个段分别编码，保留 / 分隔符 @@ -148,12 +186,10 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP Message string `json:"message"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - // 响应不是 JSON 格式，可能是空响应（某些 API 成功时返回空 body） - // 由于 HTTP 状态码是 200，视为成功 - return nil + return fmt.Errorf("解析上传响应失败: %w", err) } if result.Code != 200 { - return fmt.Errorf("上传失败: %s", result.Message) + return fmt.Errorf("上传失败: %w", &APIError{Code: result.Code, Message: result.Message}) } return nil @@ -176,7 +212,7 @@ func (c *Client) ListStorages(ctx context.Context) ([]StorageInfo, error) { if err != nil { return err } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) resp, err := c.httpClient.Do(req) if err != nil { @@ -197,7 +233,7 @@ func (c *Client) ListStorages(ctx context.Context) ([]StorageInfo, error) { } if result.Code != 200 { - return fmt.Errorf("API 错误: %s", result.Message) + return fmt.Errorf("API 错误: %w", &APIError{Code: result.Code, Message: result.Message}) } storages = result.Data.Content @@ -218,7 +254,7 @@ func (c *Client) CheckStorageHealth(ctx context.Context, storageName string) err if err != nil { return err } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) req.Header.Set("Content-Type", "application/json") resp, err := c.httpClient.Do(req) @@ -237,7 +273,7 @@ func (c *Client) CheckStorageHealth(ctx context.Context, storageName string) err } if result.Code != 200 { - return fmt.Errorf("存储不可用: %s", result.Message) + return fmt.Errorf("存储不可用: %w", &APIError{Code: result.Code, Message: result.Message}) } return nil @@ -276,7 +312,7 @@ func (c *Client) GetToken(ctx context.Context, username, password string) (strin } if result.Code != 200 { - return "", fmt.Errorf("登录失败: %s", result.Message) + return "", fmt.Errorf("登录失败: %w", &APIError{Code: result.Code, Message: result.Message}) } return result.Data.Token, nil @@ -293,7 +329,7 @@ func (c *Client) Mkdir(ctx context.Context, remotePath string) error { if err != nil { return err } - req.Header.Set("Authorization", c.token) + req.Header.Set("Authorization", c.getTokenForRequest()) req.Header.Set("Content-Type", "application/json") resp, err := c.httpClient.Do(req) @@ -313,7 +349,7 @@ func (c *Client) Mkdir(ctx context.Context, remotePath string) error { // 目录已存在也算成功 if result.Code != 200 && result.Message != "file exists" { - return fmt.Errorf("创建目录失败: %s", result.Message) + return fmt.Errorf("创建目录失败: %w", &APIError{Code: result.Code, Message: result.Message}) } return nil diff --git a/src/pkg/openlist/manager.go b/src/pkg/openlist/manager.go index 1d4e642..65a4fa1 100644 --- a/src/pkg/openlist/manager.go +++ b/src/pkg/openlist/manager.go @@ -34,6 +34,19 @@ func SetGlobalManager(m *Manager) { globalManager = m } +// ClearGlobalManagerIf 仅当全局管理器等于 expected 时才清除（CAS 语义） +// 用于异步清理路径，避免误清已被新实例替换的管理器 +// 返回是否成功清除 +func ClearGlobalManagerIf(expected *Manager) bool { + globalMu.Lock() + defer globalMu.Unlock() + if globalManager == expected { + globalManager = nil + return true + } + return false +} + // GetGlobalManager 获取全局 OpenList 管理器 func GetGlobalManager() *Manager { globalMu.RLock() @@ -197,9 +210,14 @@ func (m *Manager) saveInitialCredentials(logFile string) { return } - // 如果已经配置了用户名密码，跳过 + // 如果已经配置了用户名密码，验证有效性 if config.OpenList.Username != "" && config.OpenList.Password != "" { - return + // 用现有凭据尝试登录，验证是否有效 + testClient := NewClient(m.apiEndpoint, "") + if _, err := testClient.GetToken(context.Background(), config.OpenList.Username, config.OpenList.Password); err == nil { + return // 凭据有效，无需更新 + } + // 凭据无效（如占位值 root/root），继续从日志读取真实密码 } // 读取日志文件查找初始密码 @@ -226,10 +244,20 @@ func (m *Manager) saveInitialCredentials(logFile string) { return } + // 验证密码有效性：日志是追加模式，旧密码可能已被用户修改 + // 只有实际能登录时才保存 + client := NewClient(m.apiEndpoint, "") + if _, err := client.GetToken(context.Background(), "admin", initialPassword); err != nil { + logrus.WithError(err).Debug("初始密码验证失败，跳过保存（可能已被修改）") + return + } + // 保存到配置 - config.OpenList.Username = "admin" - config.OpenList.Password = initialPassword - if err := config.Marshal(); err != nil { + if _, err := configs.UpdateWithRetry(func(c *configs.Config) error { + c.OpenList.Username = "admin" + c.OpenList.Password = initialPassword + return nil + }, 3, 10*time.Millisecond); err != nil { logrus.WithError(err).Warn("保存 OpenList 初始密码到配置文件失败") } else { logrus.Info("OpenList 首次启动，初始密码已保存到配置文件") @@ -251,10 +279,16 @@ func (m *Manager) watchProcess() { select { case <-m.stopCh: - // 正常停止 + // 正常停止（stopInternal 已负责关闭 m.logFile） default: - // 异常退出 + // 异常退出：关闭日志文件句柄，避免 fd 泄漏 if wasRunning { + m.mu.Lock() + if m.logFile != nil { + m.logFile.Close() + m.logFile = nil + } + m.mu.Unlock() logrus.WithError(err).Warn("OpenList 进程异常退出") } } @@ -322,6 +356,7 @@ func (m *Manager) GetDataPath() string { // GetClient 获取已认证的 API 客户端 // 如果提供了 token，直接使用；否则使用用户名密码登录获取 token +// token 有效性由 withRetry 在实际 API 调用时自动处理（401/403 时自动刷新） func (m *Manager) GetClient(ctx context.Context, token, username, password string) (*Client, error) { client := NewClient(m.apiEndpoint, "") @@ -330,23 +365,10 @@ func (m *Manager) GetClient(ctx context.Context, token, username, password strin client.SetCredentials(username, password) } - // 有 token 时，验证其有效性 + // 有 token 时直接使用，不预验证（避免需要管理员权限的 API 调用） + // withRetry 会在实际调用遇到 401/403 时自动用密码刷新 token if token != "" { client.SetToken(token) - // 尝试用 token 调用一个轻量 API 验证有效性 - if _, err := client.ListStorages(ctx); err != nil { - // token 无效，尝试用密码重新登录 - if username != "" && password != "" { - logrus.Warn("配置的 OpenList token 无效，尝试用用户名密码重新登录") - newToken, loginErr := client.GetToken(ctx, username, password) - if loginErr != nil { - return nil, fmt.Errorf("OpenList token 无效且登录失败: %w", loginErr) - } - client.SetToken(newToken) - return client, nil - } - return nil, fmt.Errorf("OpenList token 无效且未配置用户名密码") - } return client, nil } diff --git a/src/pkg/ratelimit/ratelimit.go b/src/pkg/ratelimit/ratelimit.go index fd2a73b..beadcd9 100644 --- a/src/pkg/ratelimit/ratelimit.go +++ b/src/pkg/ratelimit/ratelimit.go @@ -11,6 +11,7 @@ import ( type PlatformRateLimiter struct { limiters map[string]*PlatformLimiter // 平台名称 -> 限制器 mu sync.RWMutex // 读写锁保护map + enabled bool // 是否实际应用平台级访问限制 } // PlatformLimiter 单个平台的频率限制器 @@ -21,8 +22,17 @@ type PlatformLimiter struct { inFlight chan struct{} // 限制同一平台最多一个请求在途 } -var globalRateLimiter = &PlatformRateLimiter{ - limiters: make(map[string]*PlatformLimiter), +// 平台级访问限制暂时关闭，以避免同平台请求串行排队导致新直播间长时间无法开始录制。 +// 保留限制器实现和配置结构，后续改善调度策略后可重新启用。 +const globalRateLimitEnabled = false + +var globalRateLimiter = newPlatformRateLimiter(globalRateLimitEnabled) + +func newPlatformRateLimiter(enabled bool) *PlatformRateLimiter { + return &PlatformRateLimiter{ + limiters: make(map[string]*PlatformLimiter), + enabled: enabled, + } } // GetGlobalRateLimiter 获取全局速率限制器实例 @@ -30,8 +40,17 @@ func GetGlobalRateLimiter() *PlatformRateLimiter { return globalRateLimiter } +// Enabled 返回当前限制器是否实际应用平台级访问限制。 +func (prl *PlatformRateLimiter) Enabled() bool { + return prl.enabled +} + // SetPlatformLimit 设置或更新指定平台的访问频率限制 func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec int) { + if !prl.enabled { + return + } + if intervalSec <= 0 { // 如果间隔为0或负数，移除该平台的限制 prl.mu.Lock() @@ -63,7 +82,7 @@ func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec in // EnsurePlatformLimit 在平台尚无限制时设置默认值，不覆盖已经显式配置的间隔。 // 用于房间对象早于配置持久化创建的路径，保证首次请求也不会绕过平台保护。 func (prl *PlatformRateLimiter) EnsurePlatformLimit(platform string, intervalSec int) { - if platform == "" || intervalSec <= 0 { + if !prl.enabled || platform == "" || intervalSec <= 0 { return } @@ -90,6 +109,14 @@ func (prl *PlatformRateLimiter) WaitForPlatform(platform string) { // 如果平台没有设置限制，立即返回 true // 返回 true 表示成功获取访问权限，false 表示被 context 取消 func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, platform string) bool { + if ctx.Err() != nil { + return false + } + + if !prl.enabled { + return true + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -108,6 +135,14 @@ func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, // 仅限制请求开始间隔不足以保护启动阶段：当前一个请求耗时超过最小间隔时，后续请求仍会 // 重叠执行。这里把并发槽位与开始间隔合并为一次许可，使大量直播间并发初始化时仍按平台串行。 func (prl *PlatformRateLimiter) AcquirePlatformWithContext(ctx context.Context, platform string) (release func(), ok bool) { + if ctx.Err() != nil { + return nil, false + } + + if !prl.enabled { + return func() {}, true + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -172,6 +207,10 @@ func waitForLimiterWithContext(ctx context.Context, limiter *PlatformLimiter) bo // GetPlatformNextAllowedTime 获取平台下次允许访问的时间 func (prl *PlatformRateLimiter) GetPlatformNextAllowedTime(platform string) time.Time { + if !prl.enabled { + return time.Now() + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -197,6 +236,10 @@ func (prl *PlatformRateLimiter) RemovePlatformLimit(platform string) { // GetAllPlatformLimits 获取所有平台的当前限制设置 func (prl *PlatformRateLimiter) GetAllPlatformLimits() map[string]int { + if !prl.enabled { + return map[string]int{} + } + prl.mu.RLock() defer prl.mu.RUnlock() @@ -219,6 +262,10 @@ type WaitInfo struct { // GetPlatformWaitInfo 获取指定平台的等待状态信息 func (prl *PlatformRateLimiter) GetPlatformWaitInfo(platform string) WaitInfo { + if !prl.enabled { + return WaitInfo{} + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() @@ -254,6 +301,10 @@ func (prl *PlatformRateLimiter) GetPlatformWaitInfo(platform string) WaitInfo { // ForceAccess 强制访问平台，忽略频率限制 // 返回距离上次访问的时间间隔 func (prl *PlatformRateLimiter) ForceAccess(platform string) time.Duration { + if !prl.enabled { + return 0 + } + prl.mu.RLock() limiter, exists := prl.limiters[platform] prl.mu.RUnlock() diff --git a/src/pkg/ratelimit/ratelimit_test.go b/src/pkg/ratelimit/ratelimit_test.go index 6df3932..73c4a13 100644 --- a/src/pkg/ratelimit/ratelimit_test.go +++ b/src/pkg/ratelimit/ratelimit_test.go @@ -7,7 +7,7 @@ import ( ) func TestPlatformRateLimiter(t *testing.T) { - limiter := GetGlobalRateLimiter() + limiter := newPlatformRateLimiter(true) // 设置测试平台限制：2秒间隔 limiter.SetPlatformLimit("test_platform", 2) @@ -44,7 +44,7 @@ func TestPlatformRateLimiter(t *testing.T) { } func TestPlatformRateLimiterUpdate(t *testing.T) { - limiter := GetGlobalRateLimiter() + limiter := newPlatformRateLimiter(true) // 设置初始限制 limiter.SetPlatformLimit("update_test", 3) @@ -80,6 +80,7 @@ func TestAcquirePlatformSerializesInFlightRequests(t *testing.T) { inFlight: make(chan struct{}, 1), }, }, + enabled: true, } releaseFirst, ok := limiter.AcquirePlatformWithContext(context.Background(), "test") @@ -112,7 +113,7 @@ func TestAcquirePlatformSerializesInFlightRequests(t *testing.T) { } func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) { - limiter := &PlatformRateLimiter{limiters: make(map[string]*PlatformLimiter)} + limiter := newPlatformRateLimiter(true) limiter.SetPlatformLimit("test", 5) limiter.EnsurePlatformLimit("test", 1) @@ -120,3 +121,35 @@ func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) { t.Fatalf("兜底限制覆盖了显式配置：得到 %d 秒，期望 5 秒", got) } } + +func TestGlobalPlatformRateLimiterDisabled(t *testing.T) { + limiter := GetGlobalRateLimiter() + limiter.SetPlatformLimit("test", 60) + if limiter.Enabled() { + t.Fatal("全局平台限制器应保持关闭") + } + + if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 { + t.Fatalf("全局平台限制器应保持关闭，实际限制：%v", limits) + } + + release, ok := limiter.AcquirePlatformWithContext(context.Background(), "test") + if !ok { + t.Fatal("关闭平台限制后请求应立即获准") + } + release() + + waitInfo := limiter.GetPlatformWaitInfo("test") + if waitInfo.MinIntervalSec != 0 || waitInfo.NextRequestInSec != 0 { + t.Fatalf("关闭平台限制后不应报告等待状态：%+v", waitInfo) + } + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + if limiter.WaitForPlatformWithContext(canceledCtx, "test") { + t.Fatal("关闭平台限制后仍应尊重 context 取消") + } + if release, acquired := limiter.AcquirePlatformWithContext(canceledCtx, "test"); acquired || release != nil { + t.Fatal("context 已取消时不应获取平台许可") + } +} diff --git a/src/recorders/event.go b/src/recorders/event.go index 346b953..afd9dcb 100644 --- a/src/recorders/event.go +++ b/src/recorders/event.go @@ -3,7 +3,8 @@ package recorders import "github.com/bililive-go/bililive-go/src/pkg/events" const ( - RecorderStart events.EventType = "RecorderStart" - RecorderStop events.EventType = "RecorderStop" - RecorderRestart events.EventType = "RecorderRestart" + RecorderStart events.EventType = "RecorderStart" + RecorderStop events.EventType = "RecorderStop" + RecorderRestart events.EventType = "RecorderRestart" + RecorderStopRequested events.EventType = "RecorderStopRequested" ) diff --git a/src/recorders/manager.go b/src/recorders/manager.go index 6925772..57d6ee6 100644 --- a/src/recorders/manager.go +++ b/src/recorders/manager.go @@ -73,6 +73,8 @@ type Manager interface { GetRecorderStatus(ctx context.Context, liveId types.LiveID) (map[string]interface{}, error) // GetActiveRecordingsCount 获取当前活跃的录制数量 GetActiveRecordingsCount() int + // WaitForNotifications 等待所有异步通知 goroutine 完成（关闭流程中使用） + WaitForNotifications() } // for test @@ -86,6 +88,7 @@ type manager struct { statusTicker *time.Ticker statusStopCh chan struct{} statusWg sync.WaitGroup // 用于等待广播 goroutine 退出 + notifyWg sync.WaitGroup // 跟踪异步通知 goroutine（sendAccumulatedSummary），关闭时等待 // restartingCount 追踪正在执行 CloseForRestart 的旧 recorder 数量。 // RestartRecorder 在释放锁后才执行 oldRecorder.CloseForRestart()， // 此期间 map 中只有新 recorder，但旧 recorder 仍在收尾运行。 @@ -133,6 +136,9 @@ func (m *manager) registryListener(ctx context.Context, ed events.Dispatcher) { }) ed.AddEventListener(listeners.LiveEnd, removeEvtListener) ed.AddEventListener(listeners.ListenStop, removeEvtListener) + // RecorderStopRequested 仅表示 recorder 自身遇到不可重试错误，需要被回收； + // 它不具有 LiveEnd 的全局直播状态语义，避免错误结束仍在进行的直播会话。 + ed.AddEventListener(RecorderStopRequested, removeEvtListener) } func (m *manager) Start(ctx context.Context) error { @@ -169,6 +175,12 @@ func (m *manager) Close(ctx context.Context) { inst.WaitGroup.Done() } +// WaitForNotifications 等待所有异步通知 goroutine 完成 +// 应在 PipelineManager.Close 之后调用，确保所有 pipeline 回调触发的通知已完成 +func (m *manager) WaitForNotifications() { + m.notifyWg.Wait() +} + func (m *manager) AddRecorder(ctx context.Context, live live.Live) error { m.lock.Lock() defer m.lock.Unlock() @@ -180,11 +192,15 @@ func (m *manager) addRecorderLocked(ctx context.Context, live live.Live) error { if _, ok := m.savers[live.GetLiveId()]; ok { return ErrRecorderExist } - recorder, err := newRecorder(ctx, live) + rec, err := newRecorder(ctx, live) if err != nil { return err } - m.savers[live.GetLiveId()] = recorder + // 设置通知 WaitGroup，使异步摘要发送 goroutine 可被关闭流程等待 + if recTyped, ok := rec.(*recorder); ok { + recTyped.notifyWg = &m.notifyWg + } + m.savers[live.GetLiveId()] = rec cfg := configs.GetCurrentConfig() if cfg != nil { @@ -192,12 +208,12 @@ func (m *manager) addRecorderLocked(ctx context.Context, live live.Live) error { bilisentry.GoWithContext(ctx, func(ctx context.Context) { m.cronRestart(ctx, live) }) } } - if err := recorder.Start(ctx); err != nil { + if err := rec.Start(ctx); err != nil { // Start 失败时从 map 删除并异步 Close 新 recorder，防止泄漏/僵尸实例 // 使用异步 Close 避免在持锁时执行耗时操作（如等待 ffmpeg 进程退出）， // 防止长时间阻塞其他 manager 操作 delete(m.savers, live.GetLiveId()) - bilisentry.Go(recorder.Close) + bilisentry.Go(rec.Close) return err } return nil @@ -241,6 +257,11 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error { return err } newRec := m.savers[live.GetLiveId()] + // 设置分段重启摘要屏障：阻止新 recorder 在 Pipeline 状态转移前发送摘要 + if newRecTyped, ok := newRec.(*recorder); ok { + newRecTyped.transferWg = &sync.WaitGroup{} + newRecTyped.transferWg.Add(1) + } // restartingCount 必须在释放锁之前递增，否则 Unlock 到 Add(1) 之间 // LiveEnd 可能移除新 recorder 并看到 restartingCount==0， // 导致 GetActiveRecordingsCount() 误判为"无活跃录制"触发优雅更新 @@ -274,6 +295,15 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error { } } + // 4. 转移 Pipeline 状态（确保旧分段的 Pipeline 结果不丢失） + if newRecTyped, ok := newRec.(*recorder); ok { + if oldRecTyped, ok := oldRecorder.(*recorder); ok { + newRecTyped.TransferPipelineState(oldRecTyped) + } + // 释放摘要屏障：转移已完成（或无需转移），允许新 recorder 发送摘要 + newRecTyped.transferWg.Done() + } + return nil } diff --git a/src/recorders/manager_test.go b/src/recorders/manager_test.go index f292349..87ee6f5 100644 --- a/src/recorders/manager_test.go +++ b/src/recorders/manager_test.go @@ -12,6 +12,7 @@ import ( "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" livemock "github.com/bililive-go/bililive-go/src/live/mock" + "github.com/bililive-go/bililive-go/src/pkg/events" "github.com/bililive-go/bililive-go/src/pkg/livelogger" "github.com/bililive-go/bililive-go/src/types" ) @@ -56,6 +57,27 @@ func TestManagerAddAndRemoveRecorder(t *testing.T) { assert.False(t, m.HasRecorder(context.Background(), "test")) } +// TestRecorderStopRequestedRemovesOnlyRecorder 验证 recorder 的内部停止请求会被 +// manager 消费并回收 recorder，无需伪造具有全局直播状态语义的 LiveEnd。 +func TestRecorderStopRequestedRemovesOnlyRecorder(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ctx := context.Background() + ed := events.NewDispatcher(ctx) + l := livemock.NewMockLive(ctrl) + l.EXPECT().GetLiveId().Return(types.LiveID("test")).AnyTimes() + + r := NewMockRecorder(ctrl) + r.EXPECT().Close() + m := &manager{savers: map[types.LiveID]Recorder{"test": r}} + m.registryListener(ctx, ed) + + ed.DispatchEventSync(events.NewEvent(RecorderStopRequested, l)) + + assert.False(t, m.HasRecorder(ctx, "test")) +} + // TestRestartRecorderRaceWithLiveEnd 验证 RestartRecorder 和 LiveEnd（RemoveRecorder） // 并发执行时不会产生僵尸录制器。 // diff --git a/src/recorders/recorder.go b/src/recorders/recorder.go index 4014ba9..4986d39 100644 --- a/src/recorders/recorder.go +++ b/src/recorders/recorder.go @@ -265,6 +265,41 @@ var ( _ danmakuRecorder = (*danmaku.DouyuDanmakuRecorder)(nil) ) +// pipelineSharedState 封装可在分段重启时共享的 Pipeline 状态 +// 分段重启时新旧 recorder 通过 redirect 链共享同一份状态， +// 确保旧任务的回调能正确递减新 recorder 的计数器 +type pipelineSharedState struct { + mu sync.Mutex + pendingCount int // 尚未完成的 Pipeline 任务数 + enqueued bool // 是否有 Pipeline 任务已入队 + summarySent bool // 摘要是否已发送（幂等保护） + runExited bool // run() 是否已退出（defer 中置位，回调中检查） + details []notify.RecordingFileDetail // 收集的文件详情 + sourceNames map[string]bool // 被 Pipeline 消费的原始文件名（用于合并时排除已删除的源文件） + onAllTasksDone func() // 所有任务完成时的回调（分段重启时设置） + suppressSummary bool // 为 true 时，run() 退出不推送摘要（分段重启场景），用 mu 保护 + redirectedTo atomic.Pointer[pipelineSharedState] // 分段重启时重定向到新 recorder 的共享状态 +} + +// resolveState 沿 redirect 链找到当前活跃的 pipelineSharedState。 +// 使用无锁遍历 + 锁内验证的方式，避免与 TransferPipelineState 产生死锁。 +// 返回时调用者持有 result.mu 锁。 +func resolveState(state *pipelineSharedState) *pipelineSharedState { + for { + // 无锁遍历找到链尾候选（通过 atomic.Load 避免与写入端的数据竞争） + final := state + for next := final.redirectedTo.Load(); next != nil; next = final.redirectedTo.Load() { + final = next + } + // 锁住候选状态，验证没有被并发重定向 + final.mu.Lock() + if final.redirectedTo.Load() == nil { + return final // 调用者持有 final.mu 锁 + } + final.mu.Unlock() + } +} + type recorder struct { Live live.Live ed events.Dispatcher @@ -281,6 +316,25 @@ type recorder struct { currentFileLock sync.RWMutex currentFilePath string + // closeForRestart 标记当前 recorder 正在分段重启，run() 退出时不发送摘要。 + // 在 CloseForRestart 中设置（早于 Close），run() 的 defer 中检查。 + closeForRestart atomic.Bool + + // pipelineState 封装 Pipeline 相关的共享状态 + // 使用指针，分段重启时新旧 recorder 共享同一份状态 + pipelineState *pipelineSharedState + + // transferWg 用于分段重启时的摘要屏障。 + // RestartRecorder 在创建新 recorder 后 Add(1)， + // TransferPipelineState 完成后 Done()。 + // sendAccumulatedSummary 在发送前 Wait()，确保旧分段状态已转入。 + // 非分段重启场景为 nil，Wait 是 no-op。 + transferWg *sync.WaitGroup + + // notifyWg 跟踪异步通知 goroutine（go sendAccumulatedSummary）。 + // 由 RecorderManager 持有，关闭流程中等待所有通知发送完成。 + notifyWg *sync.WaitGroup + // 当前录制的流信息（来自平台 API） currentStreamInfo *live.AvailableStreamInfo @@ -300,8 +354,7 @@ type recorder struct { // done 在 run() 退出时关闭，用于 CloseForRestart 等待 goroutine 完成 done chan struct{} - // suppressSummary 为 true 时，run() 退出不推送摘要（分段重启场景） - suppressSummary bool + // suppressSummary 已移入 pipelineSharedState，用 pipelineState.mu 保护 retryLogMu sync.Mutex lastRetryLogKey string @@ -313,14 +366,17 @@ func NewRecorder(ctx context.Context, live live.Live) (Recorder, error) { inst := instance.GetInstance(ctx) return &recorder{ - Live: live, - cache: inst.Cache, - startTime: time.Now(), - ed: inst.EventDispatcher.(events.Dispatcher), - state: begin, - stop: make(chan struct{}), - done: make(chan struct{}), - parserLock: new(sync.RWMutex), + Live: live, + cache: inst.Cache, + startTime: time.Now(), + ed: inst.EventDispatcher.(events.Dispatcher), + state: begin, + stop: make(chan struct{}), + done: make(chan struct{}), + parserLock: new(sync.RWMutex), + pipelineState: &pipelineSharedState{ + sourceNames: make(map[string]bool), + }, }, nil } @@ -381,7 +437,12 @@ func (r *recorder) tryRecord(ctx context.Context) { buf := new(bytes.Buffer) if err = tmpl.Execute(buf, info); err != nil { - r.getLogger().WithError(err).Error("failed to render filename, recording aborted") + // 文件名模板渲染失败通常是配置错误（如引用了不存在的字段），每次重试都会 + // 重复失败。请求 manager 仅回收该 recorder，避免无意义的 5s 重试风暴与 + // 日志刷屏，也不再被误计入活跃录制数；不要派发具有全局直播状态语义的 + // LiveEnd，否则会错误地提前结束仍在进行的 live session。 + r.getLogger().WithError(err).Error("failed to render filename, stopping recorder (check out_put_tmpl)") + r.ed.DispatchEvent(events.NewEvent(RecorderStopRequested, r.Live)) return } // 使用层级配置的 OutPutPath @@ -783,16 +844,26 @@ func (r *recorder) tryRecord(ctx context.Context) { return } + // 先递增 pending 计数（在入队前，避免任务被异步调度完成后回调递减到 -1） + r.pipelineState.mu.Lock() + r.pipelineState.pendingCount++ + r.pipelineState.mu.Unlock() + // 入队 Pipeline 任务 - if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles); err != nil { + if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles, r.onPipelineTaskComplete); err != nil { r.getLogger().WithError(err).Error("failed to enqueue pipeline task") + // 入队失败，回滚计数 + r.pipelineState.mu.Lock() + r.pipelineState.pendingCount-- + r.pipelineState.mu.Unlock() } else { + r.pipelineState.enqueued = true // 记录 Pipeline 阶段顺序 var stageNames []string for _, stage := range pipelineConfig.Stages { stageNames = append(stageNames, stage.Name) } - r.getLogger().Infof("pipeline task enqueued: %d files, stages: %s", len(outputFiles), stageNames) + r.getLogger().Infof("pipeline task enqueued: %d files, stages: %v", len(outputFiles), stageNames) } } } @@ -909,7 +980,18 @@ func (r *recorder) selectPreferredStream(streamInfos []*live.StreamUrlInfo) (ret func (r *recorder) run(ctx context.Context) { defer close(r.done) - defer r.sendAccumulatedSummary() + defer func() { + // 标记 run() 已退出，使回调在 remaining==0 时直接发送摘要， + // 而不是等待 r.done 通道关闭（避免 defer 执行顺序导致的漏发窗口） + r.pipelineState.mu.Lock() + r.pipelineState.runExited = true + r.pipelineState.mu.Unlock() + // 分段重启时跳过摘要：CloseForRestart 已设置此标记， + // TransferPipelineState 完成后由回调发送含完整 Pipeline 详情的摘要 + if !r.closeForRestart.Load() { + r.sendAccumulatedSummary() + } + }() const minRetryInterval = 5 * time.Second @@ -958,22 +1040,41 @@ func (r *recorder) accumulateRecordedFiles(files ...string) { // sendAccumulatedSummary 录制结束后统一推送录制文件摘要通知 // 在 run() 退出时通过 defer 调用，确保所有分段文件汇总为一条通知 func (r *recorder) sendAccumulatedSummary() { + // 分段重启屏障：等待 TransferPipelineState 完成， + // 确保旧分段的 pipelineDetails/pendingCount 已转入再发送摘要 + if r.transferWg != nil { + r.transferWg.Wait() + } + r.pipelineState.mu.Lock() + pendingCount := r.pipelineState.pendingCount + // 幂等保护：防止 defer 和回调并发双发 + if r.pipelineState.summarySent { + r.pipelineState.mu.Unlock() + return + } + // Pipeline 任务尚未全部完成，等待回调触发最终摘要 + if r.pipelineState.enqueued && pendingCount > 0 { + r.pipelineState.mu.Unlock() + r.getLogger().Infof("Pipeline 任务尚未全部完成（剩余 %d 个），延迟摘要推送", pendingCount) + return + } + // 标记摘要已发送（在实际发送前，防止并发双发） + r.pipelineState.summarySent = true + suppressSummary := r.pipelineState.suppressSummary + pipelineDetails := r.pipelineState.details + pipelineSourceNames := r.pipelineState.sourceNames + r.pipelineState.mu.Unlock() + r.recordedFilesMu.Lock() defer r.recordedFilesMu.Unlock() - if r.suppressSummary { + if suppressSummary { r.getLogger().Infof("录制摘要推送已抑制（分段重启），累积 %d 个文件将传递给新 recorder", len(r.recordedFiles)) return } - if len(r.recordedFiles) == 0 { + if len(r.recordedFiles) == 0 && len(pipelineDetails) == 0 { r.getLogger().Info("无录制文件，跳过摘要推送") return } - obj, err := r.cache.Get(r.Live) - if err != nil { - r.getLogger().WithError(err).Error("获取直播信息失败，无法推送录制摘要") - return - } - info := obj.(*live.Info) // 获取录制输出路径，用于查询剩余磁盘空间 cfg := configs.GetCurrentConfig() @@ -984,8 +1085,208 @@ func (r *recorder) sendAccumulatedSummary() { outputPath = resolved.OutPutPath } - r.getLogger().Infof("推送录制摘要：%d 个文件", len(r.recordedFiles)) - notify.SendRecordingSummary(r.getLogger(), info.HostName, r.Live.GetPlatformCNName(), r.recordedFiles, outputPath) + // 使用 Pipeline 收集的最终文件详情（如有），否则用录制累积的原始文件 + if len(pipelineDetails) > 0 { + // 合并未被 Pipeline 覆盖的录制文件（如某分段因 stages==0 未入队） + merged := append([]notify.RecordingFileDetail(nil), pipelineDetails...) + if len(r.recordedFiles) > 0 { + inPipeline := map[string]bool{} + for _, d := range pipelineDetails { + inPipeline[d.Name] = true + } + // 排除已被 Pipeline 转换并删除的源文件（如 convert_mp4 delete_source 场景） + for name := range pipelineSourceNames { + inPipeline[name] = true + } + for _, f := range r.recordedFiles { + if !inPipeline[f.Name] { + merged = append(merged, f) + } + } + // 清理无关联视频的 .ass 文件（DAA/executor 可能已删除视频但 .ass 从 recordedFiles 加入） + merged = filterOrphanedAssFromDetails(merged) + } + r.getLogger().Infof("推送 Pipeline 录制摘要：%d 个文件", len(merged)) + hostName := "" + platform := r.Live.GetPlatformCNName() + if obj, cacheErr := r.cache.Get(r.Live); cacheErr == nil { + if info, ok := obj.(*live.Info); ok { + hostName = info.HostName + } + } + notify.SendPipelineRecordingSummary( + r.getLogger(), + hostName, + platform, + r.recordedFiles, // originalFiles（allUploaded 时显示用） + merged, // finalFiles（含未入队分段的文件） + outputPath, + ) + } else { + // Pipeline 未产出有效文件详情，回退到录制累积 + if len(r.recordedFiles) == 0 { + return + } + obj, err := r.cache.Get(r.Live) + if err != nil { + r.getLogger().WithError(err).Error("获取直播信息失败，无法推送录制摘要") + return + } + info := obj.(*live.Info) + r.getLogger().Infof("推送录制摘要：%d 个文件", len(r.recordedFiles)) + notify.SendRecordingSummary(r.getLogger(), info.HostName, r.Live.GetPlatformCNName(), r.recordedFiles, outputPath) + } +} + +// onPipelineTaskComplete Pipeline 任务完成回调 +// 由 Pipeline Manager 在 executeTask 结束时调用 +func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) { + r.getLogger().Infof("Pipeline 回调触发：状态=%s, CurrentFiles=%d, LastStageFiles=%d, InitialFiles=%d", + task.Status, len(task.CurrentFiles), len(task.LastStageFiles), len(task.InitialFiles)) + // 沿 redirect 链找到当前活跃的共享状态 + // 支持连续重启 A→B→C 场景：A 的回调通过 resolveState 找到 C 的状态 + ps := resolveState(r.pipelineState) + // ps.mu 已被 resolveState 锁住 + + ps.pendingCount-- + + // 收集任务的最终文件详情 + switch task.Status { + case pipeline.PipelineStatusCompleted: + // 始终从 LastStageFiles 提取已上传的文件（含 Metadata["uploaded"] 标记） + uploadedDetails := extractUploadedDetails(task.LastStageFiles) + if len(uploadedDetails) > 0 { + ps.details = append(ps.details, uploadedDetails...) + } + // 合并本地保留的文件（CurrentFiles 中未被上传标记覆盖的文件） + uploadedNames := map[string]bool{} + for _, d := range uploadedDetails { + uploadedNames[d.Name] = true + } + for _, d := range pipeline.FileInfoToDetails(task.CurrentFiles) { + if !uploadedNames[d.Name] { + ps.details = append(ps.details, d) + } + } + // 回退：无上传标记且无保留文件时用 InitialFiles + if len(uploadedDetails) == 0 && len(task.CurrentFiles) == 0 { + for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) { + d.Uploaded = true + ps.details = append(ps.details, d) + } + } + case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled: + // 失败/取消时优先用 CurrentFiles（最后成功阶段的输出），回退到 InitialFiles + if len(task.CurrentFiles) > 0 { + ps.details = append(ps.details, pipeline.FileInfoToDetails(task.CurrentFiles)...) + } else { + ps.details = append(ps.details, pipeline.FileInfoToDetails(task.InitialFiles)...) + } + } + + // 收集被 Pipeline 转换的原始文件名（SourcePath），用于摘要合并时排除已删除的源文件 + collectSourceNames(ps.sourceNames, task.CurrentFiles, task.LastStageFiles) + + remaining := ps.pendingCount + runExited := ps.runExited + onAllDone := ps.onAllTasksDone + + ps.mu.Unlock() + + r.getLogger().Infof("Pipeline 任务完成（状态: %s），剩余 %d 个任务", task.Status, remaining) + + // 所有任务完成且录制已结束，发送统一摘要 + // 使用 goroutine 避免通知发送（Telegram/Email 等）阻塞 Pipeline 任务槽位 + if remaining == 0 { + if onAllDone != nil { + // 分段重启场景：通过共享回调触发摘要（回调内部检查 runExited） + onAllDone() + } else if runExited { + // 非分段重启场景：run() 已退出，异步发送摘要 + if r.notifyWg != nil { + r.notifyWg.Add(1) + } + go func() { + defer func() { + if r.notifyWg != nil { + r.notifyWg.Done() + } + }() + r.sendAccumulatedSummary() + }() + } + // run() 仍在运行且无回调：等待 run() 退出时由 defer 中的 sendAccumulatedSummary 发送 + } +} + +// extractUploadedDetails 从最后阶段输出文件中提取有上传标记的文件详情 +// CloudUploadStage 会为成功上传的文件设置 Metadata["uploaded"]=true +// 排除封面文件（封面不在录制摘要中展示，与 FileInfoToDetails 保持一致） +func extractUploadedDetails(files []pipeline.FileInfo) []notify.RecordingFileDetail { + var details []notify.RecordingFileDetail + for _, f := range files { + if f.Type == pipeline.FileTypeCover { + continue + } + uploaded := false + if f.Metadata != nil { + if u, ok := f.Metadata["uploaded"].(bool); ok && u { + uploaded = true + } + } + if uploaded && f.Size > 0 { + details = append(details, notify.RecordingFileDetail{ + Name: filepath.Base(f.Path), + Size: f.Size, + Uploaded: true, + }) + } + } + logrus.WithFields(logrus.Fields{ + "input_count": len(files), + "uploaded_count": len(details), + }).Debug("extractUploadedDetails") + return details +} + +// collectSourceNames 从 Pipeline 输出文件中收集被转换的原始文件名 +// 当 convert_mp4/burn_subtitles 等阶段转换文件时，输出文件的 SourcePath 指向原始文件 +// 这些原始文件可能已被 Pipeline 删除，摘要合并时需要排除，避免将已删除的源文件重新纳入 +func collectSourceNames(sourceNames map[string]bool, fileSets ...[]pipeline.FileInfo) { + for _, files := range fileSets { + for _, f := range files { + if f.SourcePath != "" { + sourceNames[filepath.Base(f.SourcePath)] = true + } + } + } +} + +// filterOrphanedAssFromDetails 从文件详情列表中移除无关联视频的 .ass 文件 +// 当 DAA/executor 删除了视频但 .ass 从 recordedFiles 加入 merged 时， +// 需要清理这些孤立的 .ass，避免通知误显为本地保留 +func filterOrphanedAssFromDetails(details []notify.RecordingFileDetail) []notify.RecordingFileDetail { + videoExts := map[string]bool{".flv": true, ".mp4": true, ".mkv": true, ".ts": true} + videoBases := map[string]bool{} + for _, d := range details { + ext := strings.ToLower(filepath.Ext(d.Name)) + if videoExts[ext] { + base := strings.TrimSuffix(d.Name, ext) + videoBases[base] = true + } + } + var result []notify.RecordingFileDetail + for _, d := range details { + ext := strings.ToLower(filepath.Ext(d.Name)) + if ext == ".ass" { + base := strings.TrimSuffix(d.Name, ".ass") + if !videoBases[base] { + continue // 无关联视频，跳过 + } + } + result = append(result, d) + } + return result } func (r *recorder) logStreamURLRetry(err error) { @@ -1120,9 +1421,12 @@ func (r *recorder) Close() { } func (r *recorder) CloseForRestart() []notify.RecordingFileDetail { - r.recordedFilesMu.Lock() - r.suppressSummary = true - r.recordedFilesMu.Unlock() + // 先设置重启标记，使 run() 的 defer 跳过 sendAccumulatedSummary， + // 避免在 TransferPipelineState 之前发送不含 Pipeline 详情的摘要 + r.closeForRestart.Store(true) + r.pipelineState.mu.Lock() + r.pipelineState.suppressSummary = true + r.pipelineState.mu.Unlock() r.Close() <-r.done // 等待 run() 完全退出，确保最后一个文件已累积 r.recordedFilesMu.Lock() @@ -1143,6 +1447,67 @@ func (r *recorder) SetInitialRecordedFiles(files []notify.RecordingFileDetail) { r.getLogger().Infof("继承上一个 recorder 的 %d 个录制文件", len(files)) } +// TransferPipelineState 从旧 recorder 转移 Pipeline 状态 +// 分段重启时由 RestartRecorder 调用，确保旧分段的 Pipeline 结果不丢失 +// 使用 redirect 链替代指针覆盖，支持连续重启 A→B→C 场景 +func (r *recorder) TransferPipelineState(old *recorder) { + r.pipelineState.mu.Lock() + defer r.pipelineState.mu.Unlock() + + // 沿 old 的 redirect 链找到最终活跃状态（可能经过多次重启 A→B→C→...） + oldState := resolveState(old.pipelineState) + // 此时 oldState.mu 已被 resolveState 锁住 + + // 合并 Pipeline 状态 + r.pipelineState.enqueued = r.pipelineState.enqueued || oldState.enqueued + r.pipelineState.pendingCount += oldState.pendingCount + // 注意：suppressSummary 不合并。该标志表示"我是被重启的 recorder，不发摘要"， + // 新 recorder 从未调用 CloseForRestart，应保持自己的 suppressSummary=false。 + // 回调通过 onAllTasksDone（含 runExited 检查）触发，不依赖 suppressSummary 传播。 + + // 合并文件详情（旧 recorder 的详情追加到新 recorder） + r.pipelineState.details = append(r.pipelineState.details, oldState.details...) + + // 合并已被 Pipeline 消费的原始文件名 + for name := range oldState.sourceNames { + r.pipelineState.sourceNames[name] = true + } + + // 设置回调：旧任务全部完成时，触发摘要发送 + // 回调与 suppressSummary 解耦：suppressSummary 是 recorder 级别的抑制标志， + // 回调是 pipeline 级别的完成通知。多跳 A→B→C 场景下，A 的回调通过 redirect + // 链解析到 C 的状态，如果只在 suppressSummary=true 时设置回调，B→C 转移时 + // B.suppressSummary=false 导致 C 没有回调，A 的摘要永远发不出。 + // runExited 检查确保只在新 recorder 已退出时发送，避免录制中提前发出摘要。 + r.pipelineState.onAllTasksDone = func() { + r.pipelineState.mu.Lock() + shouldSend := r.pipelineState.runExited + r.pipelineState.mu.Unlock() + if shouldSend { + if r.notifyWg != nil { + r.notifyWg.Add(1) + } + go func() { + defer func() { + if r.notifyWg != nil { + r.notifyWg.Done() + } + }() + r.sendAccumulatedSummary() + }() + } + // run() 未退出时不发送：新 recorder 的 defer 会在退出时检查并发送 + } + + // 关键：设置 redirect 链，使旧 recorder 及更早的 recorder 的回调 + // 能通过 resolveState 找到当前活跃的共享状态 + oldState.redirectedTo.Store(r.pipelineState) + oldState.mu.Unlock() + + r.getLogger().Infof("继承 Pipeline 状态：pending=%d, details=%d 个文件", + r.pipelineState.pendingCount, len(r.pipelineState.details)) +} + func (r *recorder) getLogger() *livelogger.LiveLogger { return r.Live.GetLogger() } diff --git a/src/recorders/recorder_test.go b/src/recorders/recorder_test.go index f0899c9..1583099 100644 --- a/src/recorders/recorder_test.go +++ b/src/recorders/recorder_test.go @@ -11,9 +11,15 @@ import ( gomock "go.uber.org/mock/gomock" "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/instance" "github.com/bililive-go/bililive-go/src/live" livemock "github.com/bililive-go/bililive-go/src/live/mock" + "github.com/bililive-go/bililive-go/src/notify" + "github.com/bililive-go/bililive-go/src/pipeline" + "github.com/bililive-go/bililive-go/src/pkg/events" + eventsmock "github.com/bililive-go/bililive-go/src/pkg/events/mock" "github.com/bililive-go/bililive-go/src/pkg/livelogger" + "github.com/bililive-go/bililive-go/src/types" ) func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) { @@ -34,19 +40,351 @@ func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) { l.EXPECT().GetStreamInfos().Return([]*live.StreamUrlInfo{{Url: streamURL}}, nil) l.EXPECT().GetLogger().Return(logger) + // 渲染失败时 recorder 应请求 manager 仅回收自身，而不是错误派发 LiveEnd。 + ed := eventsmock.NewMockDispatcher(ctrl) + ed.EXPECT().DispatchEvent(events.NewEvent(RecorderStopRequested, l)) + cache := gcache.New(1).LRU().Build() if err := cache.Set(l, &live.Info{Live: l}); err != nil { t.Fatalf("写入直播信息缓存失败: %v", err) } - r := &recorder{Live: l, cache: cache} + r := &recorder{Live: l, cache: cache, ed: ed} r.tryRecord(context.Background()) logs := logger.GetLogs() - if !strings.Contains(logs, "failed to render filename, recording aborted") { + if !strings.Contains(logs, "failed to render filename, stopping recorder") { t.Fatalf("未记录文件名渲染失败日志: %s", logs) } if !strings.Contains(logs, "render failed") { t.Fatalf("日志未保留原始模板错误: %s", logs) } } + +// newTestRecorder 创建用于测试的 recorder 实例（不启动 run goroutine） +func newTestRecorder(t *testing.T, liveId types.LiveID) *recorder { + t.Helper() + ctrl := gomock.NewController(t) + l := livemock.NewMockLive(ctrl) + l.EXPECT().GetLiveId().Return(liveId).AnyTimes() + l.EXPECT().GetLogger().Return(livelogger.New(0, logrus.Fields{"test": t.Name()})).AnyTimes() + + // 创建带 instance 的 context（NewRecorder 需要 instance.Cache 和 EventDispatcher） + inst := &instance.Instance{ + Cache: gcache.New(1).LRU().Build(), + } + ctx := context.WithValue(context.Background(), instance.Key, inst) + // NewDispatcher 会自动设置 inst.EventDispatcher + events.NewDispatcher(ctx) + + r, err := NewRecorder(ctx, l) + if err != nil { + t.Fatalf("创建 recorder 失败: %v", err) + } + return r.(*recorder) +} + +func TestResolveState_NoRedirect(t *testing.T) { + state := &pipelineSharedState{ + sourceNames: make(map[string]bool), + pendingCount: 5, + } + + resolved := resolveState(state) + if resolved != state { + t.Fatal("无 redirect 时 resolveState 应返回原始状态") + } + if resolved.pendingCount != 5 { + t.Fatalf("pendingCount 期望 5，实际 %d", resolved.pendingCount) + } + resolved.mu.Unlock() +} + +func TestResolveState_SingleRedirect(t *testing.T) { + stateA := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 3} + stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 0} + + // 模拟 A→B 重定向 + stateA.redirectedTo.Store(stateB) + + resolved := resolveState(stateA) + if resolved != stateB { + t.Fatal("单跳 redirect 应解析到 stateB") + } + resolved.mu.Unlock() +} + +func TestResolveState_MultiHopChain(t *testing.T) { + stateA := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 3} + stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 2} + stateC := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 1} + + // 模拟 A→B→C 链 + stateA.redirectedTo.Store(stateB) + stateB.redirectedTo.Store(stateC) + + resolved := resolveState(stateA) + if resolved != stateC { + t.Fatal("多跳 redirect 链应解析到 stateC") + } + resolved.mu.Unlock() +} + +func TestTransferPipelineState_ChainRedirect(t *testing.T) { + // 模拟 A→B→C 连续分段重启 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + recC := newTestRecorder(t, "test-live") + + // A 有 3 个待处理任务和一些文件详情 + recA.pipelineState.pendingCount = 3 + recA.pipelineState.enqueued = true + recA.pipelineState.details = []notify.RecordingFileDetail{ + {Name: "a.flv", Size: 100}, + } + recA.pipelineState.sourceNames["a_src.flv"] = true + // A 模拟已抑制（CloseForRestart 设置） + recA.pipelineState.suppressSummary = true + + // A → B 重启 + recB.TransferPipelineState(recA) + + // 验证 B 继承了 A 的状态 + if recB.pipelineState.pendingCount != 3 { + t.Fatalf("B pendingCount 期望 3，实际 %d", recB.pipelineState.pendingCount) + } + if len(recB.pipelineState.details) != 1 { + t.Fatalf("B details 期望 1，实际 %d", len(recB.pipelineState.details)) + } + + // B 有额外的 2 个任务 + recB.pipelineState.pendingCount += 2 // 模拟 B 自己又入队了 2 个任务 + + // B → C 重启 + recC.TransferPipelineState(recB) + + // 验证 C 继承了所有状态 + if recC.pipelineState.pendingCount != 5 { + t.Fatalf("C pendingCount 期望 5，实际 %d", recC.pipelineState.pendingCount) + } + + // 关键验证：A 的 pipelineState 通过 redirect 链应解析到 C 的状态 + resolvedA := resolveState(recA.pipelineState) + if resolvedA != recC.pipelineState { + t.Fatal("A 的状态应通过 redirect 链解析到 C 的状态") + } + resolvedA.mu.Unlock() + + // B 的 pipelineState 应直接指向 C + resolvedB := resolveState(recB.pipelineState) + if resolvedB != recC.pipelineState { + t.Fatal("B 的状态应解析到 C 的状态") + } + resolvedB.mu.Unlock() +} + +func TestOnPipelineTaskComplete_AfterChainRedirect(t *testing.T) { + // 模拟 A→B→C 后，A 的回调操作 C 的状态 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + recC := newTestRecorder(t, "test-live") + + // A 有 1 个待处理任务 + recA.pipelineState.pendingCount = 1 + recA.pipelineState.enqueued = true + recA.pipelineState.suppressSummary = true + recA.pipelineState.runExited = true + + // A → B → C + recB.TransferPipelineState(recA) + recC.TransferPipelineState(recB) + + // 模拟 C 的 run() 已退出（C 是当前活跃 recorder，runExited 是 per-recorder 的） + recC.pipelineState.mu.Lock() + recC.pipelineState.runExited = true + recC.pipelineState.mu.Unlock() + + // C 的 onAllTasksDone 回调应存在（TransferPipelineState 始终设置） + onAllDoneCalled := false + recC.pipelineState.onAllTasksDone = func() { + recC.pipelineState.mu.Lock() + shouldSend := recC.pipelineState.runExited + recC.pipelineState.mu.Unlock() + if shouldSend { + onAllDoneCalled = true + } + } + + // 模拟 A 的 Pipeline 任务完成 + task := &pipeline.PipelineTask{ + Status: pipeline.PipelineStatusCompleted, + CurrentFiles: []pipeline.FileInfo{}, + InitialFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo}, + }, + LastStageFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo, Metadata: map[string]any{"uploaded": true}}, + }, + } + + recA.onPipelineTaskComplete(task) + + // 验证：C 的 pendingCount 应为 0（A 的回调正确递减了 C 的计数） + if recC.pipelineState.pendingCount != 0 { + t.Fatalf("C pendingCount 期望 0，实际 %d（A 的回调未正确操作 C 的状态）", recC.pipelineState.pendingCount) + } + + // 验证：A 的回调收集的 details 应在 C 的状态中 + if len(recC.pipelineState.details) == 0 { + t.Fatal("C 的 details 不应为空（A 的回调应将文件详情写入 C 的状态）") + } + + // 验证：onAllTasksDone 应被触发（C 已退出） + if !onAllDoneCalled { + t.Fatal("onAllTasksDone 回调应被触发（C.runExited=true）") + } +} + +func TestExtractUploadedDetails_ExcludesCover(t *testing.T) { + files := []pipeline.FileInfo{ + {Path: "/data/video.flv", Size: 1024, Type: pipeline.FileTypeVideo, Metadata: map[string]any{"uploaded": true}}, + {Path: "/data/cover.jpg", Size: 512, Type: pipeline.FileTypeCover, Metadata: map[string]any{"uploaded": true}}, + {Path: "/data/other.txt", Size: 256, Type: pipeline.FileTypeOther, Metadata: map[string]any{"uploaded": true}}, + } + + details := extractUploadedDetails(files) + + if len(details) != 2 { + t.Fatalf("期望 2 个文件（排除封面），实际 %d", len(details)) + } + + names := map[string]bool{} + for _, d := range details { + names[d.Name] = true + } + if names["cover.jpg"] { + t.Fatal("封面文件不应出现在上传详情中") + } + if !names["video.flv"] || !names["other.txt"] { + t.Fatal("视频和其他类型文件应出现在上传详情中") + } +} + +func TestSuppressSummary_InSharedState(t *testing.T) { + // 验证 suppressSummary 在 pipelineSharedState 中，通过 mu 保护 + rec := newTestRecorder(t, "test-live") + + // 初始值应为 false + rec.pipelineState.mu.Lock() + if rec.pipelineState.suppressSummary { + t.Fatal("suppressSummary 初始值应为 false") + } + rec.pipelineState.mu.Unlock() + + // CloseForRestart 设置 suppressSummary + rec.pipelineState.mu.Lock() + rec.pipelineState.suppressSummary = true + rec.pipelineState.mu.Unlock() + + // 验证设置成功 + rec.pipelineState.mu.Lock() + if !rec.pipelineState.suppressSummary { + t.Fatal("suppressSummary 应为 true") + } + rec.pipelineState.mu.Unlock() +} + +func TestTransferPipelineState_SetRedirect(t *testing.T) { + // 验证 TransferPipelineState 设置了 redirect 链 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + + recA.pipelineState.pendingCount = 1 + recA.pipelineState.suppressSummary = true + + recB.TransferPipelineState(recA) + + // A 的状态应被重定向到 B + if recA.pipelineState.redirectedTo.Load() != recB.pipelineState { + t.Fatal("A 的 redirect 应指向 B 的状态") + } + + // resolveState 应从 A 找到 B + resolved := resolveState(recA.pipelineState) + if resolved != recB.pipelineState { + t.Fatal("resolveState(A) 应返回 B 的状态") + } + resolved.mu.Unlock() +} + +func TestOnAllTasksDone_NotSentWhileRecording(t *testing.T) { + // 验证 Bot 指出的场景：旧任务完成时新 recorder 仍在录制，摘要不应提前发送 + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + + recA.pipelineState.pendingCount = 1 + recA.pipelineState.enqueued = true + recA.pipelineState.suppressSummary = true + recA.pipelineState.runExited = true + + // A → B + recB.TransferPipelineState(recA) + + // B 仍在录制（runExited=false） + // 记录 B 的 onAllTasksDone 是否被调用 + callbackTriggered := false + originalCallback := recB.pipelineState.onAllTasksDone + recB.pipelineState.onAllTasksDone = func() { + callbackTriggered = true + originalCallback() + } + + // 模拟 A 的任务完成 + task := &pipeline.PipelineTask{ + Status: pipeline.PipelineStatusCompleted, + CurrentFiles: []pipeline.FileInfo{}, + InitialFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo}, + }, + LastStageFiles: []pipeline.FileInfo{ + {Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo}, + }, + } + recA.onPipelineTaskComplete(task) + + // 回调应被触发（remaining=0, suppressSummary=true） + if !callbackTriggered { + t.Fatal("onAllTasksDone 应被触发") + } + + // 但摘要不应被发送（B 仍在录制，runExited=false） + // 验证方式：summarySent 应仍为 false + recB.pipelineState.mu.Lock() + summarySent := recB.pipelineState.summarySent + recB.pipelineState.mu.Unlock() + if summarySent { + t.Fatal("摘要不应在新 recorder 仍在录制时发送（summarySent 应为 false）") + } +} + +func TestFinalRecorder_SummaryNotSuppressed(t *testing.T) { + // 验证最终 recorder 的摘要不被 suppressSummary 抑制 + // （suppressSummary 不应从旧 recorder 传播到新 recorder） + recA := newTestRecorder(t, "test-live") + recB := newTestRecorder(t, "test-live") + + recA.pipelineState.pendingCount = 1 + recA.pipelineState.enqueued = true + recA.pipelineState.suppressSummary = true + + // A → B + recB.TransferPipelineState(recA) + + // B 的 suppressSummary 应为 false（不从 A 继承） + recB.pipelineState.mu.Lock() + bSuppress := recB.pipelineState.suppressSummary + recB.pipelineState.mu.Unlock() + if bSuppress { + t.Fatal("最终 recorder 的 suppressSummary 应为 false（不应从旧 recorder 继承）") + } +} diff --git a/src/servers/handler.go b/src/servers/handler.go index a620efd..78ac66b 100644 --- a/src/servers/handler.go +++ b/src/servers/handler.go @@ -35,6 +35,7 @@ import ( applog "github.com/bililive-go/bililive-go/src/log" "github.com/bililive-go/bililive-go/src/pkg/livelogger" "github.com/bililive-go/bililive-go/src/pkg/memstats" + "github.com/bililive-go/bililive-go/src/pkg/metadata" "github.com/bililive-go/bililive-go/src/pkg/ratelimit" "github.com/bililive-go/bililive-go/src/pkg/utils" "github.com/bililive-go/bililive-go/src/recorders" @@ -184,7 +185,13 @@ func getLive(writer http.ResponseWriter, r *http.Request) { } // 获取平台等待状态信息 - waitInfo := ratelimit.GetGlobalRateLimiter().GetPlatformWaitInfo(platformKey) + platformRateLimiter := ratelimit.GetGlobalRateLimiter() + platformRateLimitEnabled := platformRateLimiter.Enabled() + waitInfo := platformRateLimiter.GetPlatformWaitInfo(platformKey) + platformRateLimit := 0 + if platformRateLimitEnabled { + platformRateLimit = cfg.GetPlatformMinAccessInterval(platformKey) + } // 获取调度器状态信息 var schedulerStatus *live.SchedulerStatus @@ -249,10 +256,12 @@ func getLive(writer http.ResponseWriter, r *http.Request) { "audio_only": room.AudioOnly, // 平台访问限制 - "platform_rate_limit": cfg.GetPlatformMinAccessInterval(platformKey), + "platform_rate_limit": platformRateLimit, + "platform_rate_limit_enabled": platformRateLimitEnabled, // 平台等待状态 "rate_limit_info": map[string]interface{}{ + "enabled": platformRateLimitEnabled, "waited_seconds": waitInfo.WaitedSeconds, "next_request_in_sec": waitInfo.NextRequestInSec, "min_interval_sec": waitInfo.MinIntervalSec, @@ -468,7 +477,8 @@ func parseLiveAction(writer http.ResponseWriter, r *http.Request) { case "forceRefresh": // 强制刷新：忽略平台访问频率限制，立即获取最新信息 platformKey := configs.GetPlatformKeyFromUrl(live.GetRawUrl()) - ratelimit.GetGlobalRateLimiter().ForceAccess(platformKey) + platformRateLimiter := ratelimit.GetGlobalRateLimiter() + platformRateLimiter.ForceAccess(platformKey) // 手动调用 GetInfo 获取最新信息 info, err := live.GetInfo() @@ -480,8 +490,9 @@ func parseLiveAction(writer http.ResponseWriter, r *http.Request) { } // 广播频率限制更新事件，通知前端更新倒计时 - waitInfo := ratelimit.GetGlobalRateLimiter().GetPlatformWaitInfo(platformKey) + waitInfo := platformRateLimiter.GetPlatformWaitInfo(platformKey) GetSSEHub().BroadcastRateLimitUpdate(live.GetLiveId(), map[string]interface{}{ + "enabled": platformRateLimiter.Enabled(), "waited_seconds": waitInfo.WaitedSeconds, "next_request_in_sec": waitInfo.NextRequestInSec, "min_interval_sec": waitInfo.MinIntervalSec, @@ -941,8 +952,9 @@ func getConfig(writer http.ResponseWriter, r *http.Request) { } func putConfig(writer http.ResponseWriter, r *http.Request) { + // 直接序列化当前快照即可；索引缓存不参与序列化，且共享快照不可变， + // 不能在此调用 RefreshLiveRoomIndexCache 写它的 map。 config := configs.GetCurrentConfig() - config.RefreshLiveRoomIndexCache() if err := config.Marshal(); err != nil { writeJsonWithStatusCode(writer, http.StatusBadRequest, commonResp{ ErrNo: http.StatusBadRequest, @@ -991,7 +1003,8 @@ func putRawConfig(writer http.ResponseWriter, r *http.Request) { return } oldConfig := configs.GetCurrentConfig() - oldConfig.RefreshLiveRoomIndexCache() + // 共享快照不可变：其索引缓存在生成时已由 Update 维护好， + // 这里只做只读查找，不能再调用 RefreshLiveRoomIndexCache 写它的 map。 // 继承原配置的文件路径 newConfig.File = oldConfig.File // 预先将旧配置中的 LiveId 迁移到新配置（相同 URL） @@ -1186,6 +1199,7 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) { }) return } + platformRateLimitEnabled := ratelimit.GetGlobalRateLimiter().Enabled() // 统计每个平台的直播间（只统计正在监控的） platformRooms := make(map[string][]map[string]interface{}) @@ -1268,7 +1282,7 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) { // 检查是否低于最小访问间隔 warningMessage := "" - if listeningCount > 0 && platformConfig.MinAccessIntervalSec > 0 && actualAccessInterval < float64(platformConfig.MinAccessIntervalSec) { + if platformRateLimitEnabled && listeningCount > 0 && platformConfig.MinAccessIntervalSec > 0 && actualAccessInterval < float64(platformConfig.MinAccessIntervalSec) { effectiveInterval := float64(platformConfig.MinAccessIntervalSec) * float64(listeningCount) warningMessage = fmt.Sprintf("当前设置下实际每个直播间的检测间隔约为 %.1f 秒（受最小访问间隔限制）", effectiveInterval) } @@ -1348,9 +1362,10 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) { } response := map[string]interface{}{ - "platforms": stats, - "available_platforms": availablePlatforms, - "global_interval": cfg.Interval, + "platforms": stats, + "available_platforms": availablePlatforms, + "global_interval": cfg.Interval, + "platform_rate_limit_enabled": platformRateLimitEnabled, } writeJSON(writer, response) @@ -1726,6 +1741,9 @@ func applyConfigUpdates(c *configs.Config, updates map[string]interface{}) error if deleteAllAfter, ok := cloudUpload["delete_all_after_upload"].(bool); ok { c.OnRecordFinished.CloudUpload.DeleteAllAfterUpload = deleteAllAfter } + if uploadSubtitles, ok := cloudUpload["upload_subtitles"].(bool); ok { + c.OnRecordFinished.CloudUpload.UploadSubtitles = uploadSubtitles + } } // 处理上传时机 if uploadTiming, ok := orf["upload_timing"].(string); ok { @@ -2419,6 +2437,7 @@ func getFileInfo(writer http.ResponseWriter, r *http.Request) { LastModified int64 `json:"last_modified"` Size int64 `json:"size"` SubtitleFile string `json:"subtitle_file,omitempty"` + Uploaded bool `json:"uploaded,omitempty"` } // First pass: separate ASS files and build base-name -> ASS file map @@ -2461,6 +2480,14 @@ func getFileInfo(writer http.ResponseWriter, r *http.Request) { if assName, ok := assFiles[baseName]; ok { jf.SubtitleFile = assName } + // Check if this file has been uploaded to cloud + relPath := fe.dir.Name() + if path != "" { + relPath = path + "/" + fe.dir.Name() // key 统一用正斜杠 + } + if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, relPath); err == nil && val != "" { + jf.Uploaded = true + } } jsonFiles = append(jsonFiles, jf) } @@ -2580,6 +2607,30 @@ func renameFile(writer http.ResponseWriter, r *http.Request) { } } + // 迁移上传标记（key 统一用正斜杠） + oldRel := path + newRelPath, _ := filepath.Rel(base, newAbsPath) + newRel := filepath.ToSlash(newRelPath) + if info.IsDir() { + // 目录重命名：迁移该目录下所有文件的上传标记 + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + oldPrefix := oldRel + "/" + for key, val := range allMarks { + if strings.HasPrefix(key, oldPrefix) { + suffix := key[len(oldPrefix):] + newKey := newRel + "/" + suffix + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newKey, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + // 文件重命名：迁移单个上传标记 + if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, oldRel); err == nil { + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newRel, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, oldRel) + } + } + writeJSON(writer, commonResp{Data: "OK"}) } @@ -2599,8 +2650,14 @@ func deleteFile(writer http.ResponseWriter, r *http.Request) { return } + // 记录是否为目录（删除前检查） + isDir := false + if info, err := os.Stat(absPath); err == nil { + isDir = info.IsDir() + } + // 删除关联的 ASS 弹幕文件 - if info, err := os.Stat(absPath); err == nil && !info.IsDir() { + if !isDir { assPath := strings.TrimSuffix(absPath, filepath.Ext(absPath)) + ".ass" if _, err := os.Stat(assPath); err == nil { os.Remove(assPath) @@ -2612,6 +2669,20 @@ func deleteFile(writer http.ResponseWriter, r *http.Request) { return } + // 清除上传标记 + if isDir { + // 目录：清除该目录下所有文件的上传标记 + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + prefix := path + "/" // key 统一用正斜杠 + for key := range allMarks { + if strings.HasPrefix(key, prefix) { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, path) + } + writeJSON(writer, commonResp{Data: "OK"}) } @@ -2698,6 +2769,28 @@ func batchRenameFiles(writer http.ResponseWriter, r *http.Request) { os.Rename(oldAss, newAss) } } + + // 迁移上传标记（key 统一用正斜杠） + oldRel := path + newRelPath, _ := filepath.Rel(base, newAbsPath) + newRel := filepath.ToSlash(newRelPath) + if info.IsDir() { + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + oldPrefix := oldRel + "/" + for key, val := range allMarks { + if strings.HasPrefix(key, oldPrefix) { + suffix := key[len(oldPrefix):] + newKey := newRel + "/" + suffix + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newKey, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, oldRel); err == nil { + metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newRel, val) + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, oldRel) + } + } } } @@ -2734,8 +2827,14 @@ func batchDeleteFiles(writer http.ResponseWriter, r *http.Request) { continue } + // 记录是否为目录（删除前检查） + isDir := false + if info, err := os.Stat(absPath); err == nil { + isDir = info.IsDir() + } + // 删除关联的 ASS 弹幕文件 - if info, err := os.Stat(absPath); err == nil && !info.IsDir() { + if !isDir { assPath := strings.TrimSuffix(absPath, filepath.Ext(absPath)) + ".ass" if _, err := os.Stat(assPath); err == nil { os.Remove(assPath) @@ -2745,6 +2844,18 @@ func batchDeleteFiles(writer http.ResponseWriter, r *http.Request) { if err := os.RemoveAll(absPath); err != nil { results = append(results, Result{Path: path, Success: false, Message: translateOSError(err)}) } else { + // 清除上传标记 + if isDir { + allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded) + prefix := path + "/" // key 统一用正斜杠 + for key := range allMarks { + if strings.HasPrefix(key, prefix) { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key) + } + } + } else { + metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, path) + } results = append(results, Result{Path: path, Success: true, Message: "成功"}) } } diff --git a/src/servers/handler_test.go b/src/servers/handler_test.go index f5bbfe7..fb9dc88 100644 --- a/src/servers/handler_test.go +++ b/src/servers/handler_test.go @@ -1,6 +1,7 @@ package servers import ( + "context" "encoding/json" "net/http/httptest" "testing" @@ -8,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/instance" ) func TestGetSoopLiveAuthConfigDoesNotExposeSavedPassword(t *testing.T) { @@ -32,3 +34,39 @@ func TestGetSoopLiveAuthConfigDoesNotExposeSavedPassword(t *testing.T) { _, exists := data["password"] assert.False(t, exists) } + +func TestGetPlatformStatsReportsDisabledRateLimit(t *testing.T) { + cfg := configs.NewConfig() + cfg.Interval = 1 + cfg.PlatformConfigs[configs.PlatformKeyDouyin] = configs.PlatformConfig{ + Name: "抖音", + MinAccessIntervalSec: 60, + } + cfg.LiveRooms = []configs.LiveRoom{{ + Url: "https://live.douyin.com/123456", + IsListening: true, + }} + cfg.RefreshLiveRoomIndexCache() + configs.SetCurrentConfig(cfg) + t.Cleanup(func() { configs.SetCurrentConfig(configs.NewConfig()) }) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/api/config/platforms", nil) + request = request.WithContext(context.WithValue(request.Context(), instance.Key, &instance.Instance{})) + getPlatformStats(recorder, request) + + assert.Equal(t, 200, recorder.Code) + var response struct { + PlatformRateLimitEnabled bool `json:"platform_rate_limit_enabled"` + Platforms []struct { + PlatformKey string `json:"platform_key"` + WarningMessage string `json:"warning_message"` + } `json:"platforms"` + } + assert.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &response)) + assert.False(t, response.PlatformRateLimitEnabled) + if assert.Len(t, response.Platforms, 1) { + assert.Equal(t, configs.PlatformKeyDouyin, response.Platforms[0].PlatformKey) + assert.Empty(t, response.Platforms[0].WarningMessage) + } +} diff --git a/src/servers/pipeline_handler.go b/src/servers/pipeline_handler.go index 68f16ca..d084c2d 100644 --- a/src/servers/pipeline_handler.go +++ b/src/servers/pipeline_handler.go @@ -2,6 +2,7 @@ package servers import ( "encoding/json" + "io" "net/http" "os" "path/filepath" @@ -165,6 +166,8 @@ func makePipelineCancelTaskHandler(pm *pipeline.Manager) http.HandlerFunc { } // makePipelineRetryTaskHandler 重试任务 +// 支持可选的 JSON body: {"stage_name": "cloud_upload"} +// 无 body 或 stage_name 为空时全量重试 func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -174,9 +177,29 @@ func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc { return } - if err := pm.RetryTask(id); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + // 解析可选的 JSON body + var reqBody struct { + StageName string `json:"stage_name"` + } + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil && err != io.EOF { + http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest) + return + } + } + + if reqBody.StageName != "" { + // 从指定阶段重试 + if err := pm.RetryTaskFromStage(id, reqBody.StageName); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } else { + // 全量重试（向后兼容） + if err := pm.RetryTask(id); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } w.WriteHeader(http.StatusOK) @@ -324,7 +347,7 @@ func makeBatchBurnHandler(pm *pipeline.Manager) http.HandlerFunc { } task := pipeline.NewPipelineTask(recordInfo, burnConfig, files) - if err := pm.EnqueueTask(task); err != nil { + if err := pm.EnqueueTask(task, nil); err != nil { result.Skipped = append(result.Skipped, filepath.Base(p)+" - 入队失败: "+err.Error()) continue } diff --git a/src/servers/server.go b/src/servers/server.go index e24191b..fcf7c4a 100644 --- a/src/servers/server.go +++ b/src/servers/server.go @@ -153,10 +153,11 @@ func initMux(ctx context.Context) *mux.Router { apiRoute.HandleFunc("/openlist/status", getOpenListStatus).Methods("GET") apiRoute.HandleFunc("/openlist/check-storage", checkOpenListStorageHealth).Methods("GET") - // Pipeline 任务路由 + // Pipeline 任务路由 + 手动上传路由 inst := instance.GetInstance(ctx) if pm := pipeline.GetManager(inst); pm != nil { RegisterPipelineHandlers(apiRoute, pm) + RegisterUploadHandlers(apiRoute, pm) } // OSRP 开放直播录制协议路由 diff --git a/src/servers/upload_handler.go b/src/servers/upload_handler.go new file mode 100644 index 0000000..f1dc1a6 --- /dev/null +++ b/src/servers/upload_handler.go @@ -0,0 +1,125 @@ +package servers + +import ( + "encoding/json" + "net/http" + "path/filepath" + + "github.com/gorilla/mux" + + "github.com/bililive-go/bililive-go/src/configs" + "github.com/bililive-go/bililive-go/src/pipeline" +) + +// uploadResult 手动上传的响应结构 +type uploadResult struct { + Enqueued int `json:"enqueued"` + Skipped []string `json:"skipped"` + TaskIDs []int64 `json:"task_ids"` +} + +// RegisterUploadHandlers 注册手动上传相关的 HTTP 处理器 +// 通过 Pipeline 系统执行上传，与自动上传行为完全一致 +func RegisterUploadHandlers(r *mux.Router, pm *pipeline.Manager) { + if pm == nil { + return + } + + r.HandleFunc("/file/upload", makeUploadFileHandler(pm)).Methods("POST") + r.HandleFunc("/batch/file/upload", makeBatchUploadFilesHandler(pm)).Methods("POST") +} + +// makeUploadFileHandler 单文件上传到云盘（通过 Pipeline） +func makeUploadFileHandler(pm *pipeline.Manager) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Path string `json:"path"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, commonResp{ErrMsg: "请求参数错误"}) + return + } + if req.Path == "" { + writeJSON(w, commonResp{ErrMsg: "文件路径不能为空"}) + return + } + + // 路径校验（复用 handler.go 中的 getSafePath） + cfg := configs.GetCurrentConfig() + if cfg == nil { + writeJSON(w, commonResp{ErrMsg: "配置未初始化"}) + return + } + absPath, err := getSafePath(cfg.OutPutPath, req.Path) + if err != nil { + writeJSON(w, commonResp{ErrMsg: "无效路径: " + err.Error()}) + return + } + + enqueued, skipped, taskIDs, err := pm.EnqueueUploadTask([]string{absPath}) + if err != nil { + writeJSON(w, commonResp{ErrMsg: err.Error()}) + return + } + + writeJSON(w, commonResp{Data: uploadResult{ + Enqueued: enqueued, + Skipped: skipped, + TaskIDs: taskIDs, + }}) + } +} + +// makeBatchUploadFilesHandler 批量上传文件到云盘（通过 Pipeline） +func makeBatchUploadFilesHandler(pm *pipeline.Manager) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Paths []string `json:"paths"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, commonResp{ErrMsg: "请求参数错误"}) + return + } + + // 批量路径校验 + cfg := configs.GetCurrentConfig() + if cfg == nil { + writeJSON(w, commonResp{ErrMsg: "配置未初始化"}) + return + } + + absPaths := make([]string, 0, len(req.Paths)) + skipped := []string{} + for _, p := range req.Paths { + absPath, err := getSafePath(cfg.OutPutPath, p) + if err != nil { + skipped = append(skipped, filepath.Base(p)+" - 路径越权") + continue + } + absPaths = append(absPaths, absPath) + } + + // 如果所有路径都校验失败，直接返回 + if len(absPaths) == 0 { + writeJSON(w, commonResp{Data: uploadResult{ + Enqueued: 0, + Skipped: skipped, + TaskIDs: []int64{}, + }}) + return + } + + enqueued, moreSkipped, taskIDs, err := pm.EnqueueUploadTask(absPaths) + if err != nil { + writeJSON(w, commonResp{ErrMsg: err.Error()}) + return + } + skipped = append(skipped, moreSkipped...) + + writeJSON(w, commonResp{Data: uploadResult{ + Enqueued: enqueued, + Skipped: skipped, + TaskIDs: taskIDs, + }}) + } +} diff --git a/src/webapp/src/component/config-info/CloudUploadSettings.tsx b/src/webapp/src/component/config-info/CloudUploadSettings.tsx index 4ad5712..57fb610 100644 --- a/src/webapp/src/component/config-info/CloudUploadSettings.tsx +++ b/src/webapp/src/component/config-info/CloudUploadSettings.tsx @@ -44,8 +44,8 @@ const UPLOAD_PRESETS: PresetTemplate[] = [ }, { name: '简洁归档', - description: '按平台/主播归档，文件名带日期', - template: '/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }}', + description: '按平台/主播归档，文件名带日期时间', + template: '/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02 15-04-05" }}.{{ .Ext }}', }, { name: '按房间归档', @@ -95,7 +95,7 @@ const MOCK_DATA: MockStream[] = [ const renderUploadTemplate = (template: string, stream: MockStream, room: MockStream['rooms'][0]): string => { const fileName = `[${room.date} ${room.time}][${stream.hostName}][${room.roomName}].flv`; let path = template; - path = path.replace(/\{\{ ?\.Platform ?\}\}/g, stream.platform); + path = path.replace(/\{\{ ?\.Platform ?\}\}/g, stream.platformCN); path = path.replace(/\{\{ ?\.HostName ?\}\}/g, stream.hostName); path = path.replace(/\{\{ ?\.RoomName ?\}\}/g, room.roomName); path = path.replace(/\{\{ ?\.FileName ?\}\}/g, fileName); @@ -123,28 +123,40 @@ const analyzeConfig = (config: any) => { const upload = cu.enable ?? false; const delAfter = cu.delete_after_upload ?? false; const delAllAfter = cu.delete_all_after_upload ?? false; + const uploadAss = cu.upload_subtitles ?? false; - // 文件状态：uploaded(上传), deleted(删除), kept(保留) + // 文件状态：uploaded(上传保留), uploaded_deleted(上传后删除), intermediate(中间产物,不上传), + // deleted(删除), kept(本地保留) const files: Record = {}; // 初始文件 files['video.flv'] = { ext: '.flv', desc: '原始录制视频', status: 'kept' }; files['video.ass'] = { ext: '.ass', desc: '弹幕字幕', status: 'kept' }; - // immediate 模式：先上传原始文件 + // immediate 模式：先上传原始文件（.ass 在录制结束时已存在，可一并上传） if (isImmediate && upload) { files['video.flv'].status = 'uploaded'; + if (uploadAss) { + files['video.ass'].status = 'uploaded'; + } } // 转码 if (convert) { files['video.mp4'] = { ext: '.mp4', desc: '转码后视频', status: 'kept' }; if (deleteFlv) { - // 如果已经被标记为 uploaded，改为 uploaded_deleted - if (files['video.flv'].status === 'uploaded') { + if (isImmediate && upload) { + // immediate 模式：.flv 已在 pipeline 开头被上传，转换后被标记为 Deletable 并删除 + files['video.flv'].status = 'uploaded_deleted'; + } else if (files['video.flv'].status === 'uploaded') { files['video.flv'].status = 'uploaded_deleted'; } else { - files['video.flv'].status = 'deleted'; + // after_process 模式：源文件是中间产物，不参与上传 + files['video.flv'].status = 'intermediate'; + } + } else { + if (files['video.flv'].status === 'kept') { + files['video.flv'].status = 'uploaded'; } } } @@ -153,19 +165,37 @@ const analyzeConfig = (config: any) => { if (burn) { files['video.mkv'] = { ext: '.mkv', desc: '烧录后视频', status: 'kept' }; if (burnDelSource) { - // 删除转码后的 mp4 或原始 flv const target = convert ? 'video.mp4' : 'video.flv'; - if (files[target].status === 'uploaded') { + if (isImmediate && upload && !convert) { + // immediate 模式（无转码）：.flv 已在 pipeline 开头被上传，烧录后被标记为 Deletable 并删除 + files[target].status = 'uploaded_deleted'; + } else if (files[target].status === 'uploaded') { files[target].status = 'uploaded_deleted'; } else { - files[target].status = 'deleted'; + // after_process 模式或 immediate+convert（.mp4 未被上传）：中间产物 + files[target].status = 'intermediate'; + } + } else { + const src = convert ? 'video.mp4' : 'video.flv'; + // after_process 模式：源视频保留在 BurnSubtitlesStage output 中，会被 cloud_upload 上传 + // immediate 模式：源视频在 cloud_upload 之后才创建/保留，不会被上传 + if (!isImmediate && files[src] && files[src].status === 'kept') { + files[src].status = 'uploaded'; } } if (burnDelAss) { - files['video.ass'].status = 'deleted'; + if (isImmediate && upload) { + // immediate 模式：.ass 已在 pipeline 开头被上传，烧录后被标记为 Deletable 并删除 + files['video.ass'].status = 'uploaded_deleted'; + } else { + files['video.ass'].status = 'deleted'; + } } } + // immediate 模式下，convert 和 burn 产出的文件（.mp4, .mkv）不会被上传 + // 它们在 cloud_upload 之后才创建，应保持 'kept' 状态 + // 封面 if (cover) { files['cover.jpg'] = { ext: '.jpg', desc: '视频封面', status: 'kept' }; @@ -173,9 +203,10 @@ const analyzeConfig = (config: any) => { // after_process 模式：上传最终文件 if (isAfterProcess && upload) { - // 上传第一个视频和封面 + // 上传最终产物 if (burn) { files['video.mkv'].status = 'uploaded'; + // 源视频已在烧录阶段标记为 uploaded（burnDelSource=false 时） } else if (convert) { files['video.mp4'].status = 'uploaded'; } else { @@ -184,22 +215,28 @@ const analyzeConfig = (config: any) => { if (cover) { files['cover.jpg'].status = 'uploaded'; } + // 上传弹幕字幕（需 after_process 模式，immediate 模式下 .ass 尚未生成） + if (uploadAss) { + files['video.ass'].status = 'uploaded'; + } // 删除逻辑 if (delAllAfter) { - // 删除全部 + // 删除全部：先保存已上传文件的状态，再全部标记删除，最后恢复已上传的标记 + const uploadedStatus: Record = {}; + Object.entries(files).forEach(([name, f]) => { + if (f.status === 'uploaded') uploadedStatus[name] = f.status; + }); Object.values(files).forEach(f => { f.status = 'deleted'; }); - // 但上传的文件标记为 uploaded_deleted - if (burn) files['video.mkv'].status = 'uploaded_deleted'; - else if (convert) files['video.mp4'].status = 'uploaded_deleted'; - else files['video.flv'].status = 'uploaded_deleted'; - if (cover) files['cover.jpg'].status = 'uploaded_deleted'; + // 恢复已上传文件为 uploaded_deleted + Object.entries(uploadedStatus).forEach(([name]) => { + files[name].status = 'uploaded_deleted'; + }); } else if (delAfter) { // 只删除已上传的文件 - if (burn) files['video.mkv'].status = 'uploaded_deleted'; - else if (convert) files['video.mp4'].status = 'uploaded_deleted'; - else files['video.flv'].status = 'uploaded_deleted'; - if (cover) files['cover.jpg'].status = 'uploaded_deleted'; + Object.entries(files).forEach(([, f]) => { + if (f.status === 'uploaded') f.status = 'uploaded_deleted'; + }); } } @@ -208,7 +245,10 @@ const analyzeConfig = (config: any) => { const hasVideo = Object.entries(files).some(([name, f]) => { if (name === 'video.ass') return false; const ext = f.ext.toLowerCase(); - return (ext === '.flv' || ext === '.mp4' || ext === '.mkv') && f.status !== 'deleted' && f.status !== 'uploaded_deleted'; + if (ext !== '.flv' && ext !== '.mp4' && ext !== '.mkv') return false; + // 只有 kept 或 uploaded（本地保留）的视频才算"存在" + // intermediate（中间产物）和 deleted/uploaded_deleted（已删除）不算 + return f.status === 'kept' || f.status === 'uploaded'; }); if (!hasVideo) { files['video.ass'].status = 'deleted'; @@ -228,8 +268,8 @@ const analyzeConfig = (config: any) => { } else if (f.status === 'uploaded_deleted') { uploaded.push(name); deleted.push(name); - } else if (f.status === 'deleted') { - deleted.push(name); + } else if (f.status === 'deleted' || f.status === 'intermediate') { + deleted.push(name); // intermediate 是中间产物，显示为删除 } else { kept.push(name); } @@ -273,6 +313,11 @@ const FileProcessingPreview: React.FC<{ config: any }> = ({ config }) => { } +
+
💡 说明：开启「转换后删除 FLV」或「烧录后删除源视频」后，只会上传最终成品（MP4 或 MKV）。
+
如果不开启，原始视频也会一并上传到云端（相当于云端保存了两份视频）。
+
上方预览会根据你当前的设置实时显示哪些文件会被上传、删除或保留。
+
); }; @@ -291,6 +336,27 @@ interface CloudUploadSettingsProps { const CloudUploadSettings: React.FC = ({ config, form }) => { const isEnabled = config.on_record_finished?.cloud_upload?.enable; + // 订阅表单中影响文件处理预览的字段，使预览实时反映用户编辑 + const watchedOrf = Form.useWatch('on_record_finished', form); + const watchedCloudUpload = Form.useWatch(['on_record_finished', 'cloud_upload'], form); + + // 将表单实时值合并到 config 副本中，供 FileProcessingPreview 使用 + // Form.useWatch 返回 undefined 时表示字段未被编辑，此时保留 config 中的原始值 + const liveConfig = useMemo(() => { + if (!watchedOrf && !watchedCloudUpload) return config; + return { + ...config, + on_record_finished: { + ...config.on_record_finished, + ...watchedOrf, + cloud_upload: { + ...config.on_record_finished?.cloud_upload, + ...watchedCloudUpload, + }, + }, + }; + }, [config, watchedOrf, watchedCloudUpload]); + // 互斥逻辑：开启一个时关闭另一个 const handleDeleteAfterChange = (checked: boolean) => { if (checked && form) { @@ -325,7 +391,24 @@ const CloudUploadSettings: React.FC = ({ config, form > OpenList 管理页面 {' '} - 添加网盘。 + 添加网盘。{' '} + {config.openlist?.username && config.openlist?.password && ( + <> + + (登录凭据: {config.openlist.username} / {config.openlist.password}) + {' '} + + )} + + (如无法访问，尝试{' '} + + 通过代理访问 + ) + } type="info" @@ -333,8 +416,8 @@ const CloudUploadSettings: React.FC = ({ config, form style={{ marginBottom: 16 }} /> - {/* 文件处理预览 */} - + {/* 文件处理预览 - 使用表单实时值，无需保存即可反映用户编辑 */} + @@ -369,7 +452,7 @@ const CloudUploadSettings: React.FC = ({ config, form presets={UPLOAD_PRESETS} mockData={MOCK_DATA} renderTemplate={renderUploadTemplate} - placeholder="/录播归档/{{ .Platform }}/{{ .HostName }}/{{ now | date '2006-01-02' }}/{{ .FileName }}" + placeholder={'/录播归档/{{ .Platform }}/{{ .HostName }}/{{ now | date "2006-01-02" }}/{{ .FileName }}'} showTreePreview={true} width={500} /> @@ -385,6 +468,11 @@ const CloudUploadSettings: React.FC = ({ config, form + + + + + p.has_rooms); @@ -1185,6 +1188,7 @@ const PlatformSettings: React.FC<{ platform={platform} globalConfig={globalConfig} globalInterval={global_interval} + platformRateLimitEnabled={platformRateLimitEnabled} onSave={(values) => handleSave(platform.platform_key, values)} onDelete={() => handleDelete(platform.platform_key)} loading={loading} @@ -1207,6 +1211,15 @@ const PlatformSettings: React.FC<{ showIcon style={{ marginBottom: 16 }} /> + {!platformRateLimitEnabled && ( + + )} {/* 正在监控的平台 */} {platformsWithRooms.length > 0 && ( @@ -1256,11 +1269,12 @@ const PlatformConfigForm: React.FC<{ platform: PlatformStat; globalConfig: EffectiveConfig; globalInterval: number; + platformRateLimitEnabled: boolean; onSave: (values: any) => void; onDelete: () => void; loading: boolean; onNavigateToRoom: (liveId: string) => void; -}> = ({ platform, globalConfig, globalInterval, onSave, onDelete, loading, onNavigateToRoom }) => { +}> = ({ platform, globalConfig, globalInterval, platformRateLimitEnabled, onSave, onDelete, loading, onNavigateToRoom }) => { const [form] = Form.useForm(); useEffect(() => { @@ -1340,18 +1354,31 @@ const PlatformConfigForm: React.FC<{ 0, inheritedValue: '不限制 (0)', - }} - valueDisplay={(platform.min_access_interval_sec || 0) > 0 ? platform.min_access_interval_sec : '不限制 (0)'} + } : undefined} + effectiveValue={platformRateLimitEnabled ? undefined : '当前不生效'} + valueDisplay={!platformRateLimitEnabled + ? ((platform.min_access_interval_sec || 0) > 0 + ? `已配置 ${platform.min_access_interval_sec} 秒（暂不生效）` + : '暂未启用') + : ((platform.min_access_interval_sec || 0) > 0 ? platform.min_access_interval_sec : '不限制 (0)')} > - + diff --git a/src/webapp/src/component/file-list/index.tsx b/src/webapp/src/component/file-list/index.tsx index ef5432d..3d24a05 100644 --- a/src/webapp/src/component/file-list/index.tsx +++ b/src/webapp/src/component/file-list/index.tsx @@ -11,7 +11,9 @@ import { // @ts-ignore EditOutlined, // @ts-ignore - DeleteOutlined + DeleteOutlined, + // @ts-ignore + CloudUploadOutlined } from "@ant-design/icons"; import { Link, useNavigate, useParams } from "react-router-dom"; import Utils from "../../utils/common"; @@ -479,6 +481,7 @@ type CurrentFolderFile = { last_modified: number; size: number; subtitle_file?: string; + uploaded?: boolean; } const FileList: React.FC = () => { @@ -780,6 +783,71 @@ const FileList: React.FC = () => { .catch(err => message.error("删除失败: " + err)); }; + const handleUpload = (record: CurrentFolderFile, e?: React.MouseEvent) => { + if (e) e.stopPropagation(); + let fullPath = record.name; + if (pathParam) { + fullPath = pathParam + "/" + record.name; + } + + message.loading({ content: `正在上传 ${record.name}...`, key: "upload" }); + api.uploadFile(fullPath) + .then((rsp: any) => { + if (rsp.data) { + const data = rsp.data; + if (data.enqueued > 0) { + message.success({ content: `已入队上传任务: ${record.name}`, key: "upload" }); + } else if (data.skipped && data.skipped.length > 0) { + message.error({ content: data.skipped[0] || "上传失败", key: "upload" }); + } else { + message.error({ content: rsp.err_msg || "上传失败", key: "upload" }); + } + } else { + message.error({ content: rsp.err_msg || "上传失败", key: "upload" }); + } + }) + .catch((err: any) => message.error({ content: "上传失败: " + (err.message || err), key: "upload" })); + }; + + const handleBatchUpload = () => { + if (selectedRowKeys.length === 0) return; + const paths = selectedRowKeys.map(key => { + const fileName = key.toString(); + return pathParam ? `${pathParam}/${fileName}` : fileName; + }); + + message.loading({ content: `正在上传 ${paths.length} 个文件...`, key: "batchUpload" }); + api.batchUploadFiles(paths) + .then((rsp: any) => { + if (rsp.data) { + const data = rsp.data; + const enqueued = data.enqueued || 0; + const skipped = data.skipped || []; + const failCount = skipped.length; + if (failCount === 0) { + message.success({ content: `全部 ${enqueued} 个文件已入队上传`, key: "batchUpload" }); + } else if (enqueued > 0) { + message.warning({ content: `已入队 ${enqueued} 个，跳过 ${failCount} 个`, key: "batchUpload" }); + Modal.info({ + title: '部分文件已跳过', + content: ( +
+ {skipped.map((s: string, i: number) => ( +
{s}
+ ))} +
+ ), + }); + } else { + message.error({ content: skipped[0] || "批量上传失败", key: "batchUpload" }); + } + } else { + message.error({ content: rsp.err_msg || "批量上传失败", key: "batchUpload" }); + } + }) + .catch((err: any) => message.error({ content: "批量上传失败: " + (err.message || err), key: "batchUpload" })); + }; + const handleBatchDelete = () => { if (selectedRowKeys.length === 0) return; const paths = selectedRowKeys.map(key => { @@ -1153,6 +1221,11 @@ const FileList: React.FC = () => { 无字幕 )} + {record.uploaded && ( + + 已上传 + + )} ); } @@ -1194,6 +1267,18 @@ const FileList: React.FC = () => { > 重命名 + {!record.is_folder && ( + + )} handleDelete(record)} @@ -1401,6 +1486,15 @@ const FileList: React.FC = () => { ); })()} + { // 旧的频率限制信息处理逻辑（兼容性保留） const rateLimitInfo = updateData; + const rateLimitEnabled = rateLimitInfo?.enabled + ?? currentDetail?.platform_rate_limit_enabled + ?? true; const updatedDetail = { ...currentDetail, - rate_limit_info: rateLimitInfo + platform_rate_limit_enabled: rateLimitEnabled, + platform_rate_limit: rateLimitEnabled ? currentDetail?.platform_rate_limit : 0, + rate_limit_info: { + ...rateLimitInfo, + enabled: rateLimitEnabled + } }; + if (!rateLimitEnabled) { + return { + ...prevState, + expandedDetails: { + ...prevState.expandedDetails, + [roomId]: updatedDetail + }, + countdownTimers: { + ...prevState.countdownTimers, + [roomId]: 0 + }, + lastUpdateTimes: { + ...prevState.lastUpdateTimes, + [roomId]: Date.now() + }, + refreshStatus: { + ...prevState.refreshStatus, + [roomId]: 'idle' + } + }; + } + const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0); const minIntervalSec = rateLimitInfo?.min_interval_sec || currentDetail?.platform_rate_limit || 20; const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0); @@ -1235,6 +1265,9 @@ class LiveList extends React.Component { // 优先使用 scheduler_status 来确定刷新状态 const schedulerStatus = detail.scheduler_status; const rateLimitInfo = detail.rate_limit_info; + const rateLimitEnabled = detail.platform_rate_limit_enabled + ?? rateLimitInfo?.enabled + ?? true; let initialCountdown = 0; let initialStatus: RefreshStatus = 'idle'; @@ -1249,14 +1282,14 @@ class LiveList extends React.Component { // 有下次请求计划 initialCountdown = Math.ceil(schedulerStatus.seconds_until_next_request); // 检查是否在等待平台限制 - if (rateLimitInfo?.next_request_in_sec > 0) { + if (rateLimitEnabled && rateLimitInfo?.next_request_in_sec > 0) { initialStatus = 'waiting_rate_limit'; } else { initialStatus = 'waiting_interval'; } } else if (schedulerStatus.seconds_until_next_request === 0) { // 即将发送请求或正在等待平台限制 - if (rateLimitInfo?.next_request_in_sec > 0) { + if (rateLimitEnabled && rateLimitInfo?.next_request_in_sec > 0) { initialCountdown = Math.ceil(rateLimitInfo.next_request_in_sec); initialStatus = 'waiting_rate_limit'; } else { @@ -1270,19 +1303,24 @@ class LiveList extends React.Component { } } else { // 回退到旧逻辑（兼容性） - const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0); - const minIntervalSec = rateLimitInfo?.min_interval_sec || detail.platform_rate_limit || 20; - const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0); - - if (nextRequestInSec > 0) { - initialCountdown = nextRequestInSec; - initialStatus = 'waiting_rate_limit'; - } else if (waitedSec < minIntervalSec) { - initialCountdown = minIntervalSec - waitedSec; - initialStatus = 'waiting_interval'; - } else { + if (!rateLimitEnabled) { initialCountdown = 0; initialStatus = 'idle'; + } else { + const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0); + const minIntervalSec = rateLimitInfo?.min_interval_sec || detail.platform_rate_limit || 20; + const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0); + + if (nextRequestInSec > 0) { + initialCountdown = nextRequestInSec; + initialStatus = 'waiting_rate_limit'; + } else if (waitedSec < minIntervalSec) { + initialCountdown = minIntervalSec - waitedSec; + initialStatus = 'waiting_interval'; + } else { + initialCountdown = 0; + initialStatus = 'idle'; + } } } @@ -1428,6 +1466,9 @@ class LiveList extends React.Component { const countdown = countdownTimers[record.roomId] ?? 0; const status = refreshStatus[record.roomId] ?? 'idle'; const liveId = record.roomId; + const platformRateLimitEnabled = detail?.platform_rate_limit_enabled + ?? detail?.rate_limit_info?.enabled + ?? true; // 保存 this 引用供嵌套函数使用 const component = this; @@ -1782,7 +1823,27 @@ class LiveList extends React.Component { 平台访问频率控制
- {detail.rate_limit_info ? ( + {!platformRateLimitEnabled ? ( +
+ +
+ +
+
+ ) : detail.rate_limit_info ? (
{ } }; - handleRetry = async (taskId: number) => { + handleRetry = async (taskId: number, stageName?: string) => { try { - const res = await fetch(`/api/pipeline/tasks/${taskId}/retry`, { method: 'POST' }); + const body: any = {}; + if (stageName) { + body.stage_name = stageName; + } + const res = await fetch(`/api/pipeline/tasks/${taskId}/retry`, { + method: 'POST', + headers: stageName ? { 'Content-Type': 'application/json' } : undefined, + body: stageName ? JSON.stringify(body) : undefined, + }); if (res.ok) { - message.success('任务已重新排队'); + message.success(stageName ? `从 ${this.getStageLabel(stageName)} 阶段重试` : '任务已重新排队'); this.loadData(); } else { message.error('重试失败'); @@ -291,6 +300,7 @@ class PipelineTaskList extends Component { 'extract_cover': '提取封面', 'cloud_upload': '云盘上传', 'custom_command': '自定义命令', + 'burn_subtitles': '烧录字幕', }; return labels[stageName] || stageName; }; @@ -450,6 +460,9 @@ class PipelineTaskList extends Component { const results = task.stage_results || []; if (results.length === 0) return null; + // 是否允许阶段级重试（Failed/Cancelled/Completed 状态） + const canRetryStage = task.can_retry && (task.status === 'failed' || task.status === 'cancelled' || task.status === 'completed'); + return ( {results.map((result, index) => ( @@ -461,6 +474,19 @@ class PipelineTaskList extends Component { {result.status === 'failed' && ( 失败 )} + {canRetryStage && (result.status === 'completed' || result.status === 'failed') && ( + + )} } key={index} @@ -470,7 +496,11 @@ class PipelineTaskList extends Component { {result.error_message && ( + {result.error_message} +
+ } type="error" style={{ marginBottom: 8 }} /> @@ -534,7 +564,7 @@ class PipelineTaskList extends Component {
直播间： - {task.record_info.room_name} ({task.record_info.host_name} - {task.record_info.platform}) + {task.record_info.display_name || task.record_info.room_name} ({task.record_info.host_name} - {task.record_info.platform})
)} @@ -549,7 +579,11 @@ class PipelineTaskList extends Component { {task.error_message && ( + {task.error_message} +
+ } type="error" /> )} @@ -568,7 +602,7 @@ class PipelineTaskList extends Component { 取消任务 )} - {(task.status === 'failed' || task.status === 'cancelled') && task.can_retry && ( + {(task.status === 'failed' || task.status === 'cancelled' || task.status === 'completed') && task.can_retry && ( @@ -609,7 +643,7 @@ class PipelineTaskList extends Component { width: 200, render: (_, record: PipelineTask) => ( - {record.record_info?.room_name || '-'} + {record.record_info?.display_name || record.record_info?.room_name || '-'} {record.record_info?.platform && ( {record.record_info.platform} )} diff --git a/src/webapp/src/utils/api.ts b/src/webapp/src/utils/api.ts index 183fccd..a39ff32 100644 --- a/src/webapp/src/utils/api.ts +++ b/src/webapp/src/utils/api.ts @@ -170,6 +170,22 @@ class API { return utils.requestPost(`${BASE_URL}/pipeline/batch-burn`, { paths }); } + /** + * 上传单个文件到云盘 + * @param path 文件路径 + */ + uploadFile(path: string) { + return utils.requestPost(`${BASE_URL}/file/upload`, { path }); + } + + /** + * 批量上传文件到云盘 + * @param paths 文件路径列表 + */ + batchUploadFiles(paths: string[]) { + return utils.requestPost(`${BASE_URL}/batch/file/upload`, { paths }); + } + /** * 获取Cookie列表 */ diff --git a/tests/e2e/fixtures/ffmpeg-test-config.template.yml b/tests/e2e/fixtures/ffmpeg-test-config.template.yml index cb79c3f..9b67e55 100644 --- a/tests/e2e/fixtures/ffmpeg-test-config.template.yml +++ b/tests/e2e/fixtures/ffmpeg-test-config.template.yml @@ -16,7 +16,7 @@ log: rotate_days: 7 feature: remove_symbol_other_character: false -out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{.StartTime}}' +out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}.flv' video_split_strategies: on_room_name_changed: false max_duration: 0s diff --git a/tests/e2e/fixtures/test-config.template.yml b/tests/e2e/fixtures/test-config.template.yml index 0c7b8a5..9320baf 100644 --- a/tests/e2e/fixtures/test-config.template.yml +++ b/tests/e2e/fixtures/test-config.template.yml @@ -18,7 +18,7 @@ feature: # '{{ .Live.GetPlatformCNName }}/{{ .HostName | filenameFilter }}/[{{ now | date "2006-01-02 15-04-05"}}][{{ .HostName | filenameFilter }}][{{ .RoomName | filenameFilter }}].flv' # ./平台名称/主播名字/[时间戳][主播名字][房间名字].flv # https://github.com/bililive-go/bililive-go/wiki/More-Tips -out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{.StartTime}}' +out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}.flv' video_split_strategies: on_room_name_changed: false max_duration: 0s @@ -43,7 +43,7 @@ on_record_finished: cloud_upload: enable: false storage_name: "" - upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }} + upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }} delete_after_upload: false upload_timing: after_process timeout_in_us: 60000000 diff --git a/tests/e2e/recording.spec.ts b/tests/e2e/recording.spec.ts index 77c1781..4308cac 100644 --- a/tests/e2e/recording.spec.ts +++ b/tests/e2e/recording.spec.ts @@ -1,8 +1,8 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, request as apiRequest } from '@playwright/test'; /** * 直播录制功能测试 - * + * * 使用 osrp-stream-tester 提供的 dev 直播间进行测试 * 测试添加直播间、开始/停止录制等功能 */ @@ -10,6 +10,66 @@ import { test, expect } from '@playwright/test'; // dev 测试流服务器地址 const DEV_STREAM_SERVER = 'http://127.0.0.1:8888'; const DEV_STREAM_URL = `${DEV_STREAM_SERVER}/live/test.flv`; +// bililive-go 后端地址 +const BGO_SERVER = 'http://127.0.0.1:8080'; + +interface LiveInfo { + id: string; + live_url?: string; +} + +const preExistingLiveIds = new Set(); +let baselineCaptured = false; + +// 本地运行时 Playwright 可能复用开发者已经启动的 bililive-go。先记录既有房间， +// 避免测试清理误删开发者的持久化配置。 +test.beforeAll(async () => { + const ctx = await apiRequest.newContext(); + try { + const res = await ctx.get(`${BGO_SERVER}/api/lives`); + if (!res.ok()) { + return; + } + const lives: LiveInfo[] = await res.json(); + if (!Array.isArray(lives)) { + return; + } + for (const live of lives) { + if (live.id) { + preExistingLiveIds.add(live.id); + } + } + baselineCaptured = true; + } finally { + await ctx.dispose(); + } +}); + +// 本文件会通过 UI 添加 dev 直播间；由于所有 spec 共用同一个后端进程（workers=1）， +// 若不清理，残留的监控房间会一直被判定为"在播"，导致后续 spec（如 update-*）的 +// active_recordings 断言失败。这里只删除本文件新建的 dev 直播间，既保证测试隔离， +// 也不会在复用本地服务时误删开发者已有的监控房间。 +test.afterAll(async () => { + if (!baselineCaptured) { + return; + } + const ctx = await apiRequest.newContext(); + try { + const res = await ctx.get(`${BGO_SERVER}/api/lives`); + if (res.ok()) { + const lives: LiveInfo[] = await res.json(); + if (Array.isArray(lives)) { + for (const live of lives) { + if (live.id && live.live_url === DEV_STREAM_URL && !preExistingLiveIds.has(live.id)) { + await ctx.delete(`${BGO_SERVER}/api/lives/${live.id}`); + } + } + } + } + } finally { + await ctx.dispose(); + } +}); test.describe('直播间管理测试', () => { test.beforeEach(async ({ page }) => { @@ -224,4 +284,3 @@ test.describe('紧急情况：停止所有录制', () => { expect(count).toBeGreaterThanOrEqual(0); }); }); - diff --git a/tests/e2e/update-full.spec.ts b/tests/e2e/update-full.spec.ts index ea22386..c949175 100644 --- a/tests/e2e/update-full.spec.ts +++ b/tests/e2e/update-full.spec.ts @@ -208,9 +208,13 @@ test.describe('优雅更新逻辑测试', () => { const response = await request.get('/api/update/launcher'); expect(response.ok()).toBeTruthy(); - const data = await response.json(); - // 测试环境应该没有活跃录制 - expect(data.active_recordings).toBe(0); + // 测试环境应该没有活跃录制。前序 spec 清理房间后录制器回收存在短暂延迟， + // 使用轮询等待其归零，避免时序相关的偶发失败。 + await expect.poll(async () => { + const res = await request.get('/api/update/launcher'); + const d = await res.json(); + return d.active_recordings; + }, { timeout: 10000 }).toBe(0); }); test('更新状态不影响其他 API', async ({ request }) => { diff --git a/tests/e2e/update-workflow.spec.ts b/tests/e2e/update-workflow.spec.ts index 471a713..7f7785f 100644 --- a/tests/e2e/update-workflow.spec.ts +++ b/tests/e2e/update-workflow.spec.ts @@ -254,9 +254,13 @@ test.describe('优雅更新逻辑测试', () => { const response = await request.get('/api/update/launcher'); expect(response.ok()).toBeTruthy(); - const data = await response.json(); - // 测试环境应该没有活跃录制 - expect(data.active_recordings).toBe(0); + // 测试环境应该没有活跃录制。前序 spec 清理房间后录制器回收存在短暂延迟， + // 使用轮询等待其归零，避免时序相关的偶发失败。 + await expect.poll(async () => { + const res = await request.get('/api/update/launcher'); + const d = await res.json(); + return d.active_recordings; + }, { timeout: 10000 }).toBe(0); }); test('更新状态不影响其他核心 API', async ({ request }) => {